whimsical-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Sam Ruby <ru...@apache.org>
Subject [whimsy.git] [1/1] Commit 7e797e5: LDAP cache management
Date Tue, 12 Jan 2016 17:28:35 GMT
Commit 7e797e5590c7da8294a046e6f51c9604ea65f03c:
    LDAP cache management
    * Ensure LDAP objects are subject to garbage collection
    * Provide Rack middleware that will run GC periodically and out of band


Branch: refs/heads/master
Author: Sam Ruby <rubys@intertwingly.net>
Committer: Sam Ruby <rubys@intertwingly.net>
Pusher: rubys <rubys@apache.org>

------------------------------------------------------------
asf.version                                                  | + -
lib/whimsy/asf/ldap.rb                                       | ++++++++ --
lib/whimsy/asf/rack.rb                                       | +++++++++++ -
www/secmail/config.ru                                        | + 
------------------------------------------------------------
108 changes: 101 additions, 7 deletions.
------------------------------------------------------------


diff --git a/asf.version b/asf.version
index c8aa910..c09f75a 100644
--- a/asf.version
+++ b/asf.version
@@ -1 +1 @@
-0.0.70
+0.0.71
diff --git a/lib/whimsy/asf/ldap.rb b/lib/whimsy/asf/ldap.rb
index 0f68eb1..8e46d77 100644
--- a/lib/whimsy/asf/ldap.rb
+++ b/lib/whimsy/asf/ldap.rb
@@ -1,5 +1,6 @@
 require 'wunderbar'
 require 'ldap'
+require 'weakref'
 
 module ASF
   module LDAP
@@ -107,22 +108,32 @@ def self.collection
     end
 
     def self.[] name
-      collection[name] || new(name)
+      new(name)
     end
 
     def self.find name
-      collection[name] || new(name)
+      new(name)
     end
 
     def self.new name
-      collection[name] || super
+      begin
+        object = collection[name]
+        return object.reference if object and object.weakref_alive?
+      rescue
+      end
+
+      super
     end
 
     def initialize name
-      self.class.collection[name] = self
+      self.class.collection[name] = WeakRef.new(self)
       @name = name
     end
 
+    def reference
+      self
+    end
+
     unless Object.respond_to? :id
       def id
         @name
diff --git a/lib/whimsy/asf/rack.rb b/lib/whimsy/asf/rack.rb
index 83a7fd2..362bdba 100644
--- a/lib/whimsy/asf/rack.rb
+++ b/lib/whimsy/asf/rack.rb
@@ -1,6 +1,7 @@
 require_relative '../asf.rb'
 require 'rack'
 require 'etc'
+require 'thread'
 
 module ASF
   module Auth
@@ -33,7 +34,7 @@ class << env; attr_accessor :user, :password; end
       ASF::Person.new(env.user)
     end
 
-    # Simply 'use' the following class in config.ru to limit access
+    # 'use' the following class in config.ru to limit access
     # to the application to ASF committers
     class Committers < Rack::Auth::Basic
       def initialize(app)
@@ -53,7 +54,7 @@ def call(env)
       end
     end
 
-    # Simply 'use' the following class in config.ru to limit access
+    # 'use' the following class in config.ru to limit access
     # to the application to ASF members and officers and the accounting group.
     class MembersAndOfficers < Rack::Auth::Basic
       def initialize(app)
@@ -84,6 +85,87 @@ def call(env)
     end
   end
 
+  # 'use' the following class in config.ru to automatically run
+  # Garbage Collection every 'n' requests, or 'm' minutes.
+  #
+  # This tries to run garbage collection "out of band" (i.e., between
+  # requests), and when other requests are active (which can happen
+  # with threaded servers like Puma).
+  #
+  # In addition to keeping memory usage bounded, this keeps the LDAP
+  # cache from going stale.
+  #
+  class AutoGC
+    @@background = nil
+
+    def initialize(app, frequency=100, minutes=15)
+      @app = app
+      @frequency = frequency
+      @request_count = 0
+      @queue = Queue.new
+      @mutex = Mutex.new
+
+      if defined?(PhusionPassenger)
+        # https://github.com/suyccom/yousell/blob/master/config.ru
+        # https://www.phusionpassenger.com/library/indepth/ruby/out_of_band_work.html
+        if PhusionPassenger.respond_to?(:require_passenger_lib)
+          PhusionPassenger.require_passenger_lib 'rack/out_of_band_gc'
+        else
+          # Phusion Passenger < 4.0.33
+          require 'phusion_passenger/rack/out_of_band_gc'
+        end
+        @passenger = PhusionPassenger::Rack::OutOfBandGc.new(app.count)
+      end
+
+      Thread.kill(@@background) if @@background
+
+      if minutes
+        # divide minutes by frequency and use the result to determine the
+        # time between simulated requests
+        @@background = Thread.new do
+           seconds = minutes * 60.0 / frequency
+           loop do
+             sleep seconds
+             maybe_perform_gc
+           end
+        end
+      end
+    end
+
+    def call(env)
+      @queue.push 1
+
+      if @passenger
+        @passenger.call(env)
+      else
+        # https://github.com/puma/puma/issues/450
+        status, header, body = @app.call(env)
+
+        if ary = env['rack.after_reply']
+          ary << lambda {maybe_perform_gc}
+        else
+          Thread.new {sleep 0.1; maybe_perform_gc}
+        end
+
+        [status, header, body]
+      end
+    ensure
+      @queue.pop
+    end
+
+    def maybe_perform_gc
+      @mutex.synchronize do
+        @request_count += 1
+        if @queue.empty? and @request_count >= @frequency
+          @request_count = 0
+          disabled = GC.enable
+          GC.start
+          GC.disable if disabled
+        end
+      end
+    end
+  end
+
   # Apache httpd on whimsy-vm is behind a proxy that converts https
   # requests into http requests.  Update the environment variables to
   # match.
diff --git a/www/secmail/config.ru b/www/secmail/config.ru
index ec54e0d..d45fd87 100644
--- a/www/secmail/config.ru
+++ b/www/secmail/config.ru
@@ -3,5 +3,6 @@ require File.expand_path('../server.rb', __FILE__)
 require 'whimsy/asf/rack'
 
 use ASF::HTTPS_workarounds
+use ASF::AutoGC
 
 run Sinatra::Application

Mime
View raw message