20 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_SERVER_CONTEXT_H_
21 #define NET_INSTAWEB_REWRITER_PUBLIC_SERVER_CONTEXT_H_
30 #include "net/instaweb/http/public/request_context.h"
34 #include "net/instaweb/util/public/property_cache.h"
35 #include "pagespeed/controller/central_controller.h"
36 #include "pagespeed/kernel/base/abstract_mutex.h"
37 #include "pagespeed/kernel/base/atomic_bool.h"
38 #include "pagespeed/kernel/base/basictypes.h"
39 #include "pagespeed/kernel/base/cache_interface.h"
40 #include "pagespeed/kernel/base/function.h"
41 #include "pagespeed/kernel/base/hasher.h"
42 #include "pagespeed/kernel/base/md5_hasher.h"
43 #include "pagespeed/kernel/base/ref_counted_ptr.h"
44 #include "pagespeed/kernel/base/scoped_ptr.h"
45 #include "pagespeed/kernel/base/string.h"
46 #include "pagespeed/kernel/base/string_util.h"
47 #include "pagespeed/kernel/base/thread_system.h"
48 #include "pagespeed/kernel/http/content_type.h"
49 #include "pagespeed/kernel/http/request_headers.h"
50 #include "pagespeed/kernel/http/response_headers.h"
51 #include "pagespeed/kernel/thread/queued_worker_pool.h"
52 #include "pagespeed/kernel/thread/sequence.h"
53 #include "pagespeed/kernel/util/simple_random.h"
55 namespace pagespeed {
namespace js {
struct JsTokenizerPatterns; } }
57 namespace net_instaweb {
60 class CachePropertyStore;
61 class CriticalImagesFinder;
62 class CriticalSelectorFinder;
63 class RequestProperties;
64 class ExperimentMatcher;
66 class FlushEarlyInfoFinder;
70 class NamedLockManager;
73 class RewriteDriverFactory;
74 class RewriteDriverPool;
77 class RewriteOptionsManager;
82 class StaticAssetManager;
84 class ThreadSynchronizer;
87 class UsageDataReporter;
88 class UserAgentMatcher;
90 typedef RefCountedPtr<OutputResource> OutputResourcePtr;
91 typedef std::vector<OutputResourcePtr> OutputResourceVector;
102 typedef std::pair<RewriteOptions*, bool> OptionsBoolPair;
113 static const char kCacheKeyResourceNamePrefix[];
131 const ContentType* content_type, StringPiece charset,
132 StringPiece cache_control_suffix, ResponseHeaders* header)
const;
134 void set_filename_prefix(
const StringPiece& file_prefix);
135 void set_statistics(Statistics* x) { statistics_ = x; }
136 void set_rewrite_stats(
RewriteStats* x) { rewrite_stats_ = x; }
137 void set_lock_manager(NamedLockManager* x) { lock_manager_ = x; }
138 void set_enable_property_cache(
bool enabled);
139 void set_message_handler(MessageHandler* x) { message_handler_ = x; }
141 StringPiece filename_prefix()
const {
return file_prefix_; }
142 Statistics* statistics()
const {
return statistics_; }
143 NamedLockManager* lock_manager()
const {
return lock_manager_; }
144 RewriteDriverFactory* factory()
const {
return factory_; }
145 ThreadSynchronizer* thread_synchronizer() {
146 return thread_synchronizer_.get();
148 ExperimentMatcher* experiment_matcher() {
return experiment_matcher_.get(); }
159 ResponseHeaders* headers);
170 const RewriteOptions* options,
171 const UrlNamer* url_namer,
172 StringVector* decoded_urls)
const;
174 void ComputeSignature(RewriteOptions* rewrite_options)
const;
177 Hasher*
hasher()
const {
return hasher_; }
178 const Hasher* lock_hasher()
const {
return &lock_hasher_; }
179 const Hasher* contents_hasher()
const {
return &contents_hasher_; }
180 FileSystem* file_system() {
return file_system_; }
181 void set_file_system(FileSystem* fs ) { file_system_ = fs; }
182 UrlNamer* url_namer()
const {
return url_namer_; }
183 void set_url_namer(UrlNamer* n) { url_namer_ = n; }
184 RewriteOptionsManager* rewrite_options_manager()
const {
185 return rewrite_options_manager_.get();
187 SHA1Signature* signature()
const {
return signature_; }
190 StaticAssetManager* static_asset_manager()
const {
191 return static_asset_manager_;
193 void set_static_asset_manager(StaticAssetManager* manager) {
194 static_asset_manager_ = manager;
196 Scheduler* scheduler()
const {
return scheduler_; }
197 void set_scheduler(Scheduler* s) { scheduler_ = s; }
198 bool has_default_system_fetcher()
const {
199 return default_system_fetcher_ != NULL;
201 bool has_default_distributed_fetcher() {
202 return default_distributed_fetcher_ != NULL;
209 return default_distributed_fetcher_;
216 const RewriteOptions* options,
const GoogleString& fragment,
217 CacheUrlAsyncFetcher::AsyncOpHooks* hooks, UrlAsyncFetcher* fetcher);
219 Timer* timer()
const {
return timer_; }
224 HTTPCache* http_cache()
const {
return http_cache_.get(); }
225 void set_http_cache(HTTPCache* x) { http_cache_.reset(x); }
230 PropertyCache* page_property_cache()
const {
231 return page_property_cache_.get();
234 const PropertyCache::Cohort* dom_cohort()
const {
return dom_cohort_; }
235 void set_dom_cohort(
const PropertyCache::Cohort* c) { dom_cohort_ = c; }
237 const PropertyCache::Cohort* beacon_cohort()
const {
return beacon_cohort_; }
238 void set_beacon_cohort(
const PropertyCache::Cohort* c) { beacon_cohort_ = c; }
240 const PropertyCache::Cohort* dependencies_cohort()
const {
241 return dependencies_cohort_;
243 void set_dependencies_cohort(
const PropertyCache::Cohort* c) {
244 dependencies_cohort_ = c;
247 const PropertyCache::Cohort* fix_reflow_cohort()
const {
248 return fix_reflow_cohort_;
250 void set_fix_reflow_cohort(
const PropertyCache::Cohort* c) {
251 fix_reflow_cohort_ = c;
259 return filesystem_metadata_cache_;
261 void set_filesystem_metadata_cache(CacheInterface* x) {
262 filesystem_metadata_cache_ = x;
270 void set_metadata_cache(CacheInterface* x) { metadata_cache_ = x; }
272 CriticalImagesFinder* critical_images_finder()
const {
273 return critical_images_finder_.get();
275 void set_critical_images_finder(CriticalImagesFinder* finder);
277 CriticalSelectorFinder* critical_selector_finder()
const {
278 return critical_selector_finder_.get();
280 void set_critical_selector_finder(CriticalSelectorFinder* finder);
282 FlushEarlyInfoFinder* flush_early_info_finder()
const {
283 return flush_early_info_finder_.get();
285 void set_flush_early_info_finder(FlushEarlyInfoFinder* finder);
287 UserAgentMatcher* user_agent_matcher()
const {
288 return user_agent_matcher_;
290 void set_user_agent_matcher(UserAgentMatcher* n) { user_agent_matcher_ = n; }
292 SimpleRandom* simple_random() {
293 return &simple_random_;
299 void set_store_outputs_in_file_system(
bool store) {
300 store_outputs_in_file_system_ = store;
303 RewriteStats* rewrite_stats()
const {
return rewrite_stats_; }
304 MessageHandler* message_handler()
const {
return message_handler_; }
319 Sequence* worker, Function* callback);
323 void set_signature(SHA1Signature* signature) { signature_ = signature; }
324 void set_default_system_fetcher(UrlAsyncFetcher* fetcher) {
325 default_system_fetcher_ = fetcher;
327 void set_default_distributed_fetcher(UrlAsyncFetcher* fetcher) {
328 default_distributed_fetcher_ = fetcher;
338 StringPiece user_agent,
339 const RequestContextPtr& request_context);
380 const RewriteOptions* domain_options,
381 GoogleUrl* request_url,
382 RequestHeaders* request_headers,
383 ResponseHeaders* response_headers,
384 RewriteQuery* rewrite_query);
403 RewriteOptions* domain_options,
404 RewriteOptions* query_options);
423 RewriteDriverPool* pool,
const RequestContextPtr& request_ctx);
439 RewriteDriverPool* pool, RewriteOptions* options,
440 const RequestContextPtr& request_ctx);
453 RewriteOptions* custom_options,
const RequestContextPtr& request_ctx);
469 ThreadSystem* thread_system() {
return thread_system_; }
470 UsageDataReporter* usage_data_reporter() {
return usage_data_reporter_; }
475 shutting_down_.set_value(
true);
478 bool shutting_down()
const {
479 return shutting_down_.value();
514 const OutputResourcePtr& output) {
516 output->response_headers());
521 ResponseHeaders* output_headers);
532 return low_priority_rewrite_workers_;
559 return response_headers_finalized_;
561 void set_response_headers_finalized(
bool x) {
562 response_headers_finalized_ = x;
569 return available_rewrite_drivers_.get();
576 void set_hostname(
const GoogleString& x) {
582 central_controller_.reset(controller);
585 CentralController* central_controller() {
586 return central_controller_.get();
592 ResponseHeaders* headers);
597 RewriteDriver* driver);
618 void set_cache_property_store(CachePropertyStore* p);
631 const GoogleString& cohort_name,
632 PropertyCache* pcache);
639 const GoogleString& cohort_name,
640 CacheInterface* cache,
641 PropertyCache* pcache);
647 const pagespeed::js::JsTokenizerPatterns* js_tokenizer_patterns()
const {
648 return js_tokenizer_patterns_;
659 bool should_delete, AsyncFetch* fetch,
660 RewriteOptions* options);
672 virtual GoogleString
FormatOption(StringPiece option_name, StringPiece args);
678 additional_driver_pools_.push_back(pool);
682 friend class ServerContextTest;
683 typedef std::set<RewriteDriver*> RewriteDriverSet;
686 void ReleaseRewriteDriverImpl(
RewriteDriver* rewrite_driver);
690 void ApplyRemoteConfig(
const GoogleString& config,
RewriteOptions* options);
692 void ApplyConfigLine(StringPiece linesp,
RewriteOptions* options);
697 GoogleString FetchRemoteConfig(
const GoogleString& url, int64 timeout_ms,
699 RequestContextPtr request_ctx);
702 ThreadSystem* thread_system_;
704 GoogleString file_prefix_;
705 FileSystem* file_system_;
707 scoped_ptr<RewriteOptionsManager> rewrite_options_manager_;
708 UserAgentMatcher* user_agent_matcher_;
709 Scheduler* scheduler_;
713 SHA1Signature* signature_;
714 scoped_ptr<CriticalImagesFinder> critical_images_finder_;
715 scoped_ptr<CriticalSelectorFinder> critical_selector_finder_;
716 scoped_ptr<FlushEarlyInfoFinder> flush_early_info_finder_;
721 MD5Hasher lock_hasher_;
725 MD5Hasher contents_hasher_;
727 Statistics* statistics_;
730 scoped_ptr<HTTPCache> http_cache_;
731 scoped_ptr<PropertyCache> page_property_cache_;
732 CacheInterface* filesystem_metadata_cache_;
733 CacheInterface* metadata_cache_;
735 bool store_outputs_in_file_system_;
736 bool response_headers_finalized_;
737 bool enable_property_cache_;
739 NamedLockManager* lock_manager_;
740 MessageHandler* message_handler_;
742 const PropertyCache::Cohort* dom_cohort_;
743 const PropertyCache::Cohort* beacon_cohort_;
744 const PropertyCache::Cohort* dependencies_cohort_;
745 const PropertyCache::Cohort* fix_reflow_cohort_;
752 scoped_ptr<RewriteDriverPool> available_rewrite_drivers_;
755 std::vector<RewriteDriverPool*> additional_driver_pools_;
762 RewriteDriverSet active_rewrite_drivers_;
771 bool trying_to_cleanup_rewrite_drivers_;
772 RewriteDriverSet deferred_release_rewrite_drivers_;
773 bool shutdown_drivers_called_;
779 scoped_ptr<AbstractMutex> rewrite_drivers_mutex_;
783 scoped_ptr<RewriteOptions> base_class_options_;
790 QueuedWorkerPool* html_workers_;
791 QueuedWorkerPool* rewrite_workers_;
792 QueuedWorkerPool* low_priority_rewrite_workers_;
794 AtomicBool shutting_down_;
801 scoped_ptr<ThreadSynchronizer> thread_synchronizer_;
804 scoped_ptr<ExperimentMatcher> experiment_matcher_;
809 GoogleString hostname_;
813 SimpleRandom simple_random_;
815 const pagespeed::js::JsTokenizerPatterns* js_tokenizer_patterns_;
817 scoped_ptr<CachePropertyStore> cache_property_store_;
819 scoped_ptr<CentralController> central_controller_;
Definition: url_namer.h:38
CacheUrlAsyncFetcher * CreateCustomCacheFetcher(const RewriteOptions *options, const GoogleString &fragment, CacheUrlAsyncFetcher::AsyncOpHooks *hooks, UrlAsyncFetcher *fetcher)
const PropertyCache::Cohort * AddCohort(const GoogleString &cohort_name, PropertyCache *pcache)
void set_central_controller(CentralController *controller)
Takes ownership of controller.
Definition: server_context.h:581
const CacheInterface * pcache_cache_backend()
void MergeNonCachingResponseHeaders(const ResourcePtr &input, const OutputResourcePtr &output)
Definition: server_context.h:513
void SetDefaultLongCacheHeaders(const ContentType *content_type, StringPiece charset, StringPiece cache_control_suffix, ResponseHeaders *header) const
bool GetQueryOptions(const RequestContextPtr &request_context, const RewriteOptions *domain_options, GoogleUrl *request_url, RequestHeaders *request_headers, ResponseHeaders *response_headers, RewriteQuery *rewrite_query)
Definition: static_asset_manager.h:47
RewriteOptions * GetCustomOptions(RequestHeaders *request_headers, RewriteOptions *domain_options, RewriteOptions *query_options)
virtual void ApplySessionFetchers(const RequestContextPtr &req, RewriteDriver *driver)
virtual bool ProxiesHtml() const =0
void set_shutting_down()
Definition: server_context.h:474
RewriteDriver * NewRewriteDriverFromPool(RewriteDriverPool *pool, const RequestContextPtr &request_ctx)
RewriteDriverPool * standard_rewrite_driver_pool()
Definition: server_context.h:568
void set_decoding_driver(RewriteDriver *rd)
Definition: server_context.h:622
bool IsPagespeedResource(const GoogleUrl &url) const
Is this URL a ref to a Pagespeed resource?
void TryLockForCreation(NamedLock *creation_lock, Function *callback)
Attempt to obtain a named lock without blocking. Return true if we do so.
void set_timer(Timer *timer)
Note: doesn't take ownership.
Definition: server_context.h:222
void ShowCacheHandler(Format format, StringPiece url, StringPiece ua, bool should_delete, AsyncFetch *fetch, RewriteOptions *options)
void LockForCreation(NamedLock *creation_lock, Sequence *worker, Function *callback)
NamedLock * MakeCreationLock(const GoogleString &name)
const RewriteFilter * FindFilterForDecoding(const StringPiece &id) const
RewriteDriver * NewUnmanagedRewriteDriver(RewriteDriverPool *pool, RewriteOptions *options, const RequestContextPtr &request_ctx)
virtual void PostInitHook()
To set up AdminSite for SystemServerContext.
Definition: rewrite_driver_pool.h:34
QueuedWorkerPool * html_workers()
Pool of worker-threads that can be used to handle html-parsing.
Definition: server_context.h:524
static const char kResourceEtagValue[]
Definition: server_context.h:112
static GoogleString ShowCacheForm(StringPiece user_agent)
virtual GoogleString FormatOption(StringPiece option_name, StringPiece args)
bool DecodeUrlGivenOptions(const GoogleUrl &url, const RewriteOptions *options, const UrlNamer *url_namer, StringVector *decoded_urls) const
See RewriteDriver::DecodeUrl.
static const int64 kCacheTtlForMismatchedContentMs
Short lifetime for generated resources fetched with mismatching hash.
Definition: server_context.h:107
RequestProperties * NewRequestProperties()
Makes a new RequestProperties.
void SetRewriteOptionsManager(RewriteOptionsManager *rom)
Takes ownership of RewriteOptionsManager.
void GetRemoteOptions(RewriteOptions *remote_options, bool on_startup)
static bool IsExcludedAttribute(const char *attribute)
QueuedWorkerPool * low_priority_rewrite_workers()
Definition: server_context.h:531
GoogleString GetRewriteOptionsSignatureHash(const RewriteOptions *options)
bool store_outputs_in_file_system()
Definition: server_context.h:298
bool response_headers_finalized() const
Definition: server_context.h:558
void reset_global_options(RewriteOptions *options)
void MakePagePropertyCache(PropertyStore *property_store)
Creates PagePropertyCache object with the provided PropertyStore object.
RewriteOptions * NewOptions()
Makes a new, empty set of RewriteOptions.
Definition: rewrite_driver.h:98
void ManageRewriteDriverPool(RewriteDriverPool *pool)
Definition: server_context.h:677
Definition: server_context.h:100
void ApplyInputCacheControl(const ResourceVector &inputs, ResponseHeaders *headers)
void DeleteCacheOnDestruction(CacheInterface *cache)
QueuedWorkerPool * rewrite_workers()
Pool of worker-threads that can be used to handle resource rewriting.
Definition: server_context.h:527
CacheInterface * metadata_cache() const
Definition: server_context.h:269
Definition: usage_data_reporter.h:35
const GoogleString & hostname() const
Returns the current server hostname.
Definition: server_context.h:573
static const char kStatisticsGroup[]
Default statistics group name.
Definition: server_context.h:116
void ReleaseRewriteDriver(RewriteDriver *rewrite_driver)
Hasher * hasher() const
Definition: server_context.h:177
bool HandleBeacon(StringPiece body, StringPiece user_agent, const RequestContextPtr &request_context)
CacheInterface * filesystem_metadata_cache() const
Definition: server_context.h:258
UrlAsyncFetcher * DefaultSystemFetcher()
Definition: server_context.h:206
void AddOriginalContentLengthHeader(const ResourceVector &inputs, ResponseHeaders *headers)
Collects a few specific statistics variables related to Rewriting.
Definition: rewrite_stats.h:35
Definition: rewrite_options.h:84
RewriteOptions * global_options()
size_t num_active_rewrite_drivers()
static const int64 kGeneratedMaxAgeMs
The lifetime for cache-extended generated resources, in milliseconds.
Definition: server_context.h:105
RewriteDriver * NewCustomRewriteDriver(RewriteOptions *custom_options, const RequestContextPtr &request_ctx)
void set_hasher(Hasher *hasher)
Setters should probably only be used in testing.
Definition: server_context.h:322
RewriteDriver * NewRewriteDriver(const RequestContextPtr &request_ctx)
Definition: url_async_fetcher.h:33
virtual PropertyStore * CreatePropertyStore(CacheInterface *cache_backend)
Creates CachePropertyStore object which will be used by PagePropertyCache.
Definition: http_cache.h:48
const PropertyCache::Cohort * AddCohortWithCache(const GoogleString &cohort_name, CacheInterface *cache, PropertyCache *pcache)
Definition: rewrite_driver_factory.h:70