rewrite_driver.h

Go to the documentation of this file.

/*
 * Copyright 2010 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http:///www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


#ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
#define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_

#include <map>
#include <set>
#include <vector>

#include "base/logging.h"
#include "net/instaweb/http/public/cache_url_async_fetcher.h"
#include "net/instaweb/http/public/http_cache.h"
#include "net/instaweb/http/public/request_context.h"
#include "net/instaweb/http/public/url_async_fetcher.h"
#include "net/instaweb/rewriter/cached_result.pb.h"
#include "net/instaweb/rewriter/public/critical_images_finder.h"
#include "net/instaweb/rewriter/public/critical_selector_finder.h"
#include "net/instaweb/rewriter/public/csp.h"
#include "net/instaweb/rewriter/public/downstream_cache_purger.h"
#include "net/instaweb/rewriter/public/inline_attribute_slot.h"
#include "net/instaweb/rewriter/public/inline_resource_slot.h"
#include "net/instaweb/rewriter/public/output_resource.h"
#include "net/instaweb/rewriter/public/output_resource_kind.h"
#include "net/instaweb/rewriter/public/resource.h"
#include "net/instaweb/rewriter/public/resource_namer.h"
#include "net/instaweb/rewriter/public/resource_slot.h"
#include "net/instaweb/rewriter/public/rewrite_context.h"
#include "net/instaweb/rewriter/public/rewrite_options.h"
#include "net/instaweb/rewriter/public/scan_filter.h"
#include "net/instaweb/rewriter/public/server_context.h"
#include "net/instaweb/rewriter/public/srcset_slot.h"
#include "pagespeed/kernel/base/abstract_mutex.h"
#include "pagespeed/kernel/base/atomic_bool.h"
#include "pagespeed/kernel/base/basictypes.h"
#include "pagespeed/kernel/base/function.h"
#include "pagespeed/kernel/base/printf_format.h"
#include "pagespeed/kernel/base/proto_util.h"
#include "pagespeed/kernel/base/scoped_ptr.h"
#include "pagespeed/kernel/base/string.h"
#include "pagespeed/kernel/base/string_util.h"
#include "pagespeed/kernel/base/thread_annotations.h"
#include "pagespeed/kernel/base/thread_system.h"
#include "pagespeed/kernel/base/writer.h"
#include "pagespeed/kernel/html/html_element.h"
#include "pagespeed/kernel/html/html_filter.h"
#include "pagespeed/kernel/html/html_node.h"
#include "pagespeed/kernel/html/html_parse.h"
#include "pagespeed/kernel/http/content_type.h"
#include "pagespeed/kernel/http/google_url.h"
#include "pagespeed/kernel/http/request_headers.h"
#include "pagespeed/kernel/http/response_headers.h"
#include "pagespeed/kernel/http/user_agent_matcher.h"
#include "pagespeed/kernel/thread/queued_worker_pool.h"
#include "pagespeed/kernel/thread/scheduler.h"
#include "pagespeed/kernel/thread/sequence.h"
#include "pagespeed/kernel/util/categorized_refcount.h"
#include "pagespeed/kernel/util/url_segment_encoder.h"
#include "pagespeed/opt/http/property_cache.h"

namespace net_instaweb {

class AbstractLogRecord;
class AsyncFetch;
class CommonFilter;
class DebugFilter;
class DependencyTracker;
class DomStatsFilter;
class DomainRewriteFilter;
class FallbackPropertyPage;
class FileSystem;
class FlushEarlyInfo;
class HtmlWriterFilter;
class MessageHandler;
class RequestProperties;
class RequestTrace;
class RewriteDriverPool;
class RewriteFilter;
class Statistics;
class UrlLeftTrimFilter;
class UrlNamer;

class RewriteDriver : public HtmlParse {
 public:
  enum CssResolutionStatus {
    kWriteFailed,
    kNoResolutionNeeded,
    kSuccess
  };

  enum WaitMode {
    kNoWait,  
    kWaitForCompletion,    
    kWaitForCachedRender,  
    kWaitForShutDown       
  };

  enum XhtmlStatus {
    kXhtmlUnknown,
    kIsXhtml,
    kIsNotXhtml
  };

  enum InlineAuthorizationPolicy {
    kInlineUnauthorizedResources,
    kInlineOnlyAuthorizedResources
  };

  enum IntendedFor {
    kIntendedForInlining,
    kIntendedForGeneral
  };

  static const char kDomCohort[];
  static const char kBeaconCohort[];
  static const char kDependenciesCohort[];

  static const char kLastRequestTimestamp[];
  static const char kParseSizeLimitExceeded[];
  static const char kSubresourcesPropertyName[];
  static const char kStatusCodePropertyName[];

  RewriteDriver(MessageHandler* message_handler,
                FileSystem* file_system,
                UrlAsyncFetcher* url_async_fetcher);

  virtual ~RewriteDriver();

  RewriteDriver* Clone();

  void Clear();

  static void InitStats(Statistics* statistics);

  static void Initialize();
  static void Terminate();

  static GoogleString DeadlineExceededMessage(StringPiece filter_name);

  void SetServerContext(ServerContext* server_context);

  bool MayCacheExtendCss() const;
  bool MayCacheExtendImages() const;
  bool MayCacheExtendPdfs() const;
  bool MayCacheExtendScripts() const;

  const GoogleString& user_agent() const { return user_agent_; }

  const RequestProperties* request_properties() const {
    return request_properties_.get();
  }

  void ClearRequestProperties();

  bool write_property_cache_dom_cohort() const {
    return write_property_cache_dom_cohort_;
  }
  void set_write_property_cache_dom_cohort(bool x) {
    write_property_cache_dom_cohort_ = x;
  }

  static PropertyCache::CohortVector GetCohortList(
      const PropertyCache* pcache, const RewriteOptions* options,
      const ServerContext* server_context);

  void PropertyCacheSetupDone();

  RequestContextPtr request_context() { return request_context_; }
  void set_request_context(const RequestContextPtr& x);

  RequestTrace* trace_context();

  void TracePrintf(const char* fmt, ...);
  void TraceLiteral(const char* literal);
  void TraceString(const GoogleString& s);

  ResponseHeaders* mutable_response_headers() {
    return flush_occurred_ ? NULL : response_headers_;
  }

  const ResponseHeaders* response_headers() {
    return response_headers_;
  }

  void set_response_headers_ptr(ResponseHeaders* headers) {
    response_headers_ = headers;
  }

  void SetRequestHeaders(const RequestHeaders& headers);

  const RequestHeaders* request_headers() const {
    return request_headers_.get();
  }

  UserAgentMatcher* user_agent_matcher() const {
    DCHECK(server_context() != NULL);
    return server_context()->user_agent_matcher();
  }

  void AddFilters();

  void AddOwnedEarlyPreRenderFilter(HtmlFilter* filter);

  void PrependOwnedPreRenderFilter(HtmlFilter* filter);
  void AppendOwnedPreRenderFilter(HtmlFilter* filter);
  void AppendUnownedPreRenderFilter(HtmlFilter* filter);

  void AddOwnedPostRenderFilter(HtmlFilter* filter);
  void AddUnownedPostRenderFilter(HtmlFilter* filter);

  void AppendRewriteFilter(RewriteFilter* filter);

  void PrependRewriteFilter(RewriteFilter* filter);

  void AddResourceUrlClaimant(ResourceUrlClaimant* claimant);

  void SetWriter(Writer* writer);

  Writer* writer() const { return writer_; }

  bool FetchResource(const StringPiece& url, AsyncFetch* fetch);

  void FetchInPlaceResource(const GoogleUrl& gurl, bool proxy_mode,
                            AsyncFetch* async_fetch);

  bool FetchOutputResource(const OutputResourcePtr& output_resource,
                           RewriteFilter* filter,
                           AsyncFetch* async_fetch);

  OutputResourcePtr DecodeOutputResource(const GoogleUrl& url,
                                         RewriteFilter** filter) const;

  bool DecodeOutputResourceName(const GoogleUrl& url,
                                const RewriteOptions* options_to_use,
                                const UrlNamer* url_namer,
                                ResourceNamer* name_out,
                                OutputResourceKind* kind_out,
                                RewriteFilter** filter_out) const;

  bool LookupMetadataForOutputResource(
      StringPiece url,
      GoogleString* error_out,
      RewriteContext::CacheLookupResultCallback* callback);

  bool DecodeUrl(const GoogleUrl& url,
                 StringVector* decoded_urls) const;

  bool DecodeUrlGivenOptions(const GoogleUrl& url,
                             const RewriteOptions* options,
                             const UrlNamer* url_namer,
                             StringVector* decoded_urls) const;

  FileSystem* file_system() { return file_system_; }
  UrlAsyncFetcher* async_fetcher() { return url_async_fetcher_; }

  void SetSessionFetcher(UrlAsyncFetcher* f);

  CacheUrlAsyncFetcher* CreateCacheFetcher();
  CacheUrlAsyncFetcher* CreateCacheOnlyFetcher();

  ServerContext* server_context() const { return server_context_; }
  Statistics* statistics() const;

  void set_custom_options(RewriteOptions* options) {
    set_options_for_pool(NULL, options);
  }

  void set_options_for_pool(RewriteDriverPool* pool, RewriteOptions* options) {
    controlling_pool_ = pool;
    options_.reset(options);
  }

  RewriteDriverPool* controlling_pool() { return controlling_pool_; }

  const RewriteOptions* options() const { return options_.get(); }

  virtual bool StartParseId(const StringPiece& url, const StringPiece& id,
                            const ContentType& content_type);

  virtual void FinishParse();

  void FinishParseAsync(Function* callback);

  void InfoAt(const RewriteContext* context,
              const char* msg, ...) INSTAWEB_PRINTF_FORMAT(3, 4);

  bool GenerateOutputResourceNameAndUrl(
    const UrlSegmentEncoder* encoder,
    const ResourceContext* data,
    const ResourcePtr& input_resource,
    GoogleString* name,
    GoogleUrl* mapped_gurl,
    GoogleString* failure_reason);


  OutputResourcePtr CreateOutputResourceFromResource(
      const char* filter_id,
      const UrlSegmentEncoder* encoder,
      const ResourceContext* data,
      const ResourcePtr& input_resource,
      OutputResourceKind kind,
      GoogleString* failure_reason);

  OutputResourcePtr CreateOutputResourceWithPath(
      const StringPiece& mapped_path, const StringPiece& unmapped_path,
      const StringPiece& base_url, const StringPiece& filter_id,
      const StringPiece& name, OutputResourceKind kind,
      GoogleString* failure_reason);

  void PopulateResourceNamer(
    const StringPiece& filter_id,
    const StringPiece& name,
    ResourceNamer* full_name);

  OutputResourcePtr CreateOutputResourceWithUnmappedUrl(
      const GoogleUrl& unmapped_gurl, const StringPiece& filter_id,
      const StringPiece& name, OutputResourceKind kind,
      GoogleString* failure_reason);

  OutputResourcePtr CreateOutputResourceWithMappedPath(
      const StringPiece& mapped_path, const StringPiece& unmapped_path,
      const StringPiece& filter_id, const StringPiece& name,
      OutputResourceKind kind, GoogleString* failure_reason) {
    return CreateOutputResourceWithPath(mapped_path, unmapped_path,
                                        decoded_base_url_.AllExceptLeaf(),
                                        filter_id, name, kind, failure_reason);
  }

  OutputResourcePtr CreateOutputResourceWithPath(
      const StringPiece& path, const StringPiece& filter_id,
      const StringPiece& name, OutputResourceKind kind,
      GoogleString* failure_reason) {
    return CreateOutputResourceWithPath(path, path, path, filter_id, name,
                                        kind, failure_reason);
  }

  enum class InputRole {
    kScript,
    kStyle,
    kImg,
    kUnknown,
    kReconstruction,
  };

  ResourcePtr CreateInputResource(const GoogleUrl& input_url,
                                  InputRole role,
                                  bool* is_authorized);

  ResourcePtr CreateInputResource(
      const GoogleUrl& input_url,
      InlineAuthorizationPolicy inline_authorization_policy,
      IntendedFor intended_for,
      InputRole role,
      bool* is_authorized);

  ResourcePtr CreateInputResourceAbsoluteUncheckedForTestsOnly(
      const StringPiece& absolute_url);

  bool IsResourceUrlClaimed(const GoogleUrl& url) const;

  bool MatchesBaseUrl(const GoogleUrl& input_url) const;

  bool MayRewriteUrl(const GoogleUrl& domain_url,
                     const GoogleUrl& input_url,
                     InlineAuthorizationPolicy inline_authorization_policy,
                     IntendedFor intended_for,
                     bool* is_authorized_domain) const;

  const GoogleUrl& base_url() const { return base_url_; }

  StringPiece fetch_url() const { return fetch_url_; }

  const GoogleUrl& decoded_base_url() const { return decoded_base_url_; }
  StringPiece decoded_base() const { return decoded_base_url_.Spec(); }

  bool IsHttps() const { return google_url().SchemeIs("https"); }

  const UrlSegmentEncoder* default_encoder() const { return &default_encoder_; }

  RewriteFilter* FindFilter(const StringPiece& id) const;

  bool refs_before_base() const { return refs_before_base_; }
  bool other_base_problem() const { return other_base_problem_; }

  void set_refs_before_base() { refs_before_base_ = true; }

  void set_other_base_problem() { other_base_problem_ = true; }

  StringPiece containing_charset() { return containing_charset_; }
  void set_containing_charset(const StringPiece charset) {
    charset.CopyToString(&containing_charset_);
  }

  HtmlResourceSlotPtr GetSlot(const ResourcePtr& resource,
                              HtmlElement* elt,
                              HtmlElement::Attribute* attr);

  InlineResourceSlotPtr GetInlineSlot(const ResourcePtr& resource,
                                      HtmlCharactersNode* char_node);

  InlineAttributeSlotPtr GetInlineAttributeSlot(
      const ResourcePtr& resource, HtmlElement* element,
      HtmlElement::Attribute* attribute);

  SrcSetSlotCollectionPtr GetSrcSetSlotCollection(
      CommonFilter* filter, HtmlElement* element, HtmlElement::Attribute* attr);

  bool InitiateRewrite(RewriteContext* rewrite_context)
      LOCKS_EXCLUDED(rewrite_mutex());
  void InitiateFetch(RewriteContext* rewrite_context);

  void RewriteComplete(RewriteContext* rewrite_context, RenderOp permit_render);

  void ReportSlowRewrites(int num);

  void Cleanup();

  void AddUserReference();

  GoogleString ToString(bool show_detached_contexts) const
      LOCKS_EXCLUDED(rewrite_mutex());
  GoogleString ToStringLockHeld(bool show_detached_contexts) const
      EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
  void PrintState(bool show_detached_contexts); 
  void PrintStateToErrorLog(bool show_detached_contexts); 

  void WaitForCompletion();

  void WaitForShutDown();

  void BoundedWaitFor(WaitMode mode, int64 timeout_ms)
      LOCKS_EXCLUDED(rewrite_mutex());

  void set_fully_rewrite_on_flush(bool x) {
    fully_rewrite_on_flush_ = x;
  }

  bool fully_rewrite_on_flush() const {
    return fully_rewrite_on_flush_;
  }

  void set_fast_blocking_rewrite(bool x) {
    fast_blocking_rewrite_ = x;
  }

  bool fast_blocking_rewrite() const {
    return fast_blocking_rewrite_;
  }

  void EnableBlockingRewrite(RequestHeaders* request_headers);

  void set_externally_managed(bool x) { externally_managed_ = x; }

  void DetachFetch();

  void DetachedFetchComplete();

  void FetchComplete();

  void DeleteRewriteContext(RewriteContext* rewrite_context);

  int rewrite_deadline_ms() { return options()->rewrite_deadline_ms(); }

  void set_max_page_processing_delay_ms(int x) {
    max_page_processing_delay_ms_ = x;
  }
  int max_page_processing_delay_ms() { return max_page_processing_delay_ms_; }

  void set_device_type(UserAgentMatcher::DeviceType x) { device_type_ = x; }
  UserAgentMatcher::DeviceType device_type() const { return device_type_; }

  RewriteContext* RegisterForPartitionKey(const GoogleString& partition_key,
                                          RewriteContext* candidate);

  void DeregisterForPartitionKey(
      const GoogleString& partition_key, RewriteContext* candidate);

  void RequestFlush() { flush_requested_ = true; }
  bool flush_requested() const { return flush_requested_; }

  void ExecuteFlushIfRequested();

  void ExecuteFlushIfRequestedAsync(Function* callback);

  virtual void Flush();

  void FlushAsync(Function* done);

  void AddRewriteTask(Function* task);

  void AddLowPriorityRewriteTask(Function* task);

  QueuedWorkerPool::Sequence* html_worker() { return html_worker_; }
  Sequence* rewrite_worker();
  Scheduler::Sequence* scheduler_sequence() {
    return scheduler_sequence_.get();
  }

  QueuedWorkerPool::Sequence* low_priority_rewrite_worker() {
    return low_priority_rewrite_worker_;
  }

  void RunTasksOnRequestThread();

  void SwitchToQueuedWorkerPool() EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());

  Scheduler* scheduler() { return scheduler_; }

  DomainRewriteFilter* domain_rewriter() { return domain_rewriter_.get(); }
  UrlLeftTrimFilter* url_trim_filter() { return url_trim_filter_.get(); }

  CssResolutionStatus ResolveCssUrls(const GoogleUrl& input_css_base,
                                     const StringPiece& output_css_base,
                                     const StringPiece& contents,
                                     Writer* writer,
                                     MessageHandler* handler);

  bool ShouldAbsolutifyUrl(const GoogleUrl& input_base,
                           const GoogleUrl& output_base,
                           bool* proxy_mode) const;

  void UpdatePropertyValueInDomCohort(
      AbstractPropertyPage* page,
      StringPiece property_name,
      StringPiece property_value);

  PropertyPage* property_page() const;

  FallbackPropertyPage* fallback_property_page() const {
    return fallback_property_page_;
  }

  PropertyPage* origin_property_page() const;

  void set_property_page(PropertyPage* page);
  void set_fallback_property_page(FallbackPropertyPage* page);
  void set_unowned_fallback_property_page(FallbackPropertyPage* page);
  void set_origin_property_page(PropertyPage* page);

  bool is_critical_images_beacon_enabled();

  CriticalImagesInfo* critical_images_info() const {
    return critical_images_info_.get();
  }

  CriticalSelectorInfo* critical_selector_info() {
    return critical_selector_info_.get();
  }

  void set_critical_selector_info(CriticalSelectorInfo* info) {
    critical_selector_info_.reset(info);
  }

  void set_critical_images_info(CriticalImagesInfo* critical_images_info) {
    critical_images_info_.reset(critical_images_info);
  }

  bool FlattenCssImportsEnabled() const {
    return (options()->Enabled(RewriteOptions::kFlattenCssImports) ||
            (!options()->Forbidden(RewriteOptions::kFlattenCssImports) &&
             (options()->Enabled(RewriteOptions::kPrioritizeCriticalCss) ||
              options()->Enabled(RewriteOptions::kComputeCriticalCss))));
  }

  int num_inline_preview_images() const { return num_inline_preview_images_; }

  void increment_num_inline_preview_images();

  void IncrementAsyncEventsCount();

  void DecrementAsyncEventsCount();

  void IncrementRenderBlockingAsyncEventsCount();

  void DecrementRenderBlockingAsyncEventsCount();

  XhtmlStatus MimeTypeXhtmlStatus();

  void set_is_lazyload_script_flushed(bool x) {
    is_lazyload_script_flushed_ = x;
  }
  bool is_lazyload_script_flushed() const {
    return is_lazyload_script_flushed_; }

  FlushEarlyInfo* flush_early_info();

  DependencyTracker* dependency_tracker() const {
    return dependency_tracker_.get();
  }

  bool DebugMode() const { return options()->Enabled(RewriteOptions::kDebug); }

  void InsertDebugComment(StringPiece unescaped_message, HtmlNode* node);
  void InsertDebugComments(
      const protobuf::RepeatedPtrField<GoogleString>& unescaped_messages,
      HtmlElement* element);
  void InsertUnauthorizedDomainDebugComment(StringPiece url,
                                            InputRole role,
                                            HtmlElement* element);

  GoogleString GenerateUnauthorizedDomainDebugComment(
      const GoogleUrl& gurl, InputRole role);

  AbstractLogRecord* log_record();

  DomStatsFilter* dom_stats_filter() const {
    return dom_stats_filter_;
  }

  bool can_rewrite_resources() const { return can_rewrite_resources_; }

  bool is_nested() const { return is_nested_; }

  bool Write(const ResourceVector& inputs,
             const StringPiece& contents,
             const ContentType* type,
             StringPiece charset,
             OutputResource* output);

  void set_defer_instrumentation_script(bool x) {
    defer_instrumentation_script_ = x;
  }
  bool defer_instrumentation_script() const {
    return defer_instrumentation_script_;
  }

  void set_num_initiated_rewrites(int64 x) {
    ScopedMutex lock(rewrite_mutex());
    num_initiated_rewrites_ = x;
  }
  int64 num_initiated_rewrites() const {
    ScopedMutex lock(rewrite_mutex());
    return num_initiated_rewrites_;
  }
  void set_num_detached_rewrites(int64 x) {
    ScopedMutex lock(rewrite_mutex());
    num_detached_rewrites_ = x;
  }
  int64 num_detached_rewrites() const {
    ScopedMutex lock(rewrite_mutex());
    return num_detached_rewrites_;
  }

  void set_pagespeed_query_params(StringPiece x) {
    x.CopyToString(&pagespeed_query_params_);
  }
  StringPiece pagespeed_query_params() const {
    return pagespeed_query_params_;
  }

  void set_pagespeed_option_cookies(StringPiece x) {
    x.CopyToString(&pagespeed_option_cookies_);
  }
  StringPiece pagespeed_option_cookies() const {
    return pagespeed_option_cookies_;
  }

  const GoogleString& CacheFragment() const;

  bool SetOrClearPageSpeedOptionCookies(const GoogleUrl& gurl,
                                        ResponseHeaders* response_headers);

  bool Decode(StringPiece leaf, ResourceNamer* resource_namer) const;

  bool filters_added() const { return filters_added_; }
  bool has_html_writer_filter() const {
    return html_writer_filter_.get() != nullptr;
  }

  void SetIsAmpDocument(bool is_amp);
  bool is_amp_document() const { return is_amp_; }

  const CspContext& content_security_policy() const { return csp_context_; }
  CspContext* mutable_content_security_policy() { return &csp_context_; }
  bool IsLoadPermittedByCsp(const GoogleUrl& url, InputRole role);
  bool IsLoadPermittedByCsp(const GoogleUrl& url, CspDirective role);

 protected:
  virtual void DetermineFiltersBehaviorImpl();

 private:
  friend class RewriteContext;
  friend class RewriteDriverTest;
  friend class RewriteTestBase;
  friend class ServerContextTest;

  typedef std::map<GoogleString, RewriteFilter*> StringFilterMap;

  void CheckForCompletionAsync(WaitMode wait_mode, int64 timeout_ms,
                               Function* done)
      EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());

  void TryCheckForCompletion(WaitMode wait_mode, int64 end_time_ms,
                             Function* done)
      EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());

  bool IsDone(WaitMode wait_mode, bool deadline_reached)
      EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());

  bool WaitForPendingAsyncEvents(WaitMode wait_mode) {
    return wait_mode == kWaitForShutDown ||
        (fully_rewrite_on_flush_ && !fast_blocking_rewrite_);
  }

  void FlushAsyncDone(int num_rewrites, Function* callback);

  int64 ComputeCurrentFlushWindowRewriteDelayMs();

  void QueueFlushAsyncDone(int num_rewrites, Function* callback);

  void QueueFinishParseAfterFlush(Function* user_callback);
  void FinishParseAfterFlush(Function* user_callback);

  bool RewritesComplete() const EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());

  void SetBaseUrlIfUnset(const StringPiece& new_base);

  void SetBaseUrlForFetch(const StringPiece& url);

  void SetDecodedUrlFromBase();

  AbstractMutex* rewrite_mutex() const LOCK_RETURNED(scheduler_->mutex()) {
    return scheduler_->mutex();
  }

  virtual void ParseTextInternal(const char* content, int size);

  bool ShouldSkipParsing();

  int SignatureLength() const;

  friend class ScanFilter;

  void RegisterRewriteFilter(RewriteFilter* filter);

  void EnableRewriteFilter(const char* id);

  ResourcePtr CreateInputResourceUnchecked(const GoogleUrl& gurl,
                                           bool is_authorized_domain);

  void AddPreRenderFilters();
  void AddPostRenderFilters();

  bool DecodeOutputResourceNameHelper(const GoogleUrl& url,
                                      const RewriteOptions* options_to_use,
                                      const UrlNamer* url_namer,
                                      ResourceNamer* name_out,
                                      OutputResourceKind* kind_out,
                                      RewriteFilter** filter_out,
                                      GoogleString* url_base,
                                      StringVector* urls) const;

  void WriteDomCohortIntoPropertyCache();

  CacheUrlAsyncFetcher* CreateCustomCacheFetcher(UrlAsyncFetcher* base_fetcher);

  void PossiblyPurgeCachedResponseAndReleaseDriver();

  void LogStats();

  bool PrepareShouldSignal() EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
  void SignalIfRequired(bool result_of_prepare_should_signal)
      EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());

  void CleanupRequestThread();

  bool base_was_set_;

  bool refs_before_base_;

  bool other_base_problem_;

  GoogleString containing_charset_;

  void PopulateRequestContext();

  bool filters_added_;
  bool externally_managed_;

  enum RefCategory {
    kRefUser,  
    kRefParsing,  

    kRefPendingRewrites,

    kRefDetachedRewrites,

    kRefDeletingRewrites,

    kRefFetchUserFacing,

    kRefFetchBackground,

    kRefAsyncEvents,

    kRefRenderBlockingAsyncEvents,

    kNumRefCategories
  };

  friend class CategorizedRefcount<RewriteDriver, RefCategory>;

  CategorizedRefcount<RewriteDriver, RefCategory> ref_counts_;

  void LastRefRemoved();
  StringPiece RefCategoryName(RefCategory cat);

  void DropReference(RefCategory cat);

  bool release_driver_;

  WaitMode waiting_ GUARDED_BY(rewrite_mutex());

  bool waiting_deadline_reached_ GUARDED_BY(rewrite_mutex());

  bool fully_rewrite_on_flush_;

  bool fast_blocking_rewrite_;

  bool flush_requested_;
  bool flush_occurred_;

  bool is_lazyload_script_flushed_;

  bool write_property_cache_dom_cohort_;

  GoogleUrl base_url_;

  GoogleUrl decoded_base_url_;

  GoogleString fetch_url_;

  GoogleString user_agent_;

  LazyBool should_skip_parsing_;

  StringFilterMap resource_filter_map_;

  ResponseHeaders* response_headers_;

  scoped_ptr<const RequestHeaders> request_headers_;

  int status_code_; 

  typedef std::vector<RewriteContext*> RewriteContextVector;
  RewriteContextVector rewrites_; 

  int max_page_processing_delay_ms_;

  typedef std::set<RewriteContext*> RewriteContextSet;

  RewriteContextSet initiated_rewrites_ GUARDED_BY(rewrite_mutex());

  int64 num_initiated_rewrites_ GUARDED_BY(rewrite_mutex());

  int64 num_detached_rewrites_ GUARDED_BY(rewrite_mutex());

  RewriteContextSet detached_rewrites_ GUARDED_BY(rewrite_mutex());

  int possibly_quick_rewrites_ GUARDED_BY(rewrite_mutex());

  RewriteContextVector fetch_rewrites_;

  FileSystem* file_system_;
  ServerContext* server_context_;
  Scheduler* scheduler_;
  UrlAsyncFetcher* default_url_async_fetcher_; 

  UrlAsyncFetcher* url_async_fetcher_;

  std::vector<UrlAsyncFetcher*> owned_url_async_fetchers_;

  DomStatsFilter* dom_stats_filter_;
  scoped_ptr<HtmlWriterFilter> html_writer_filter_;

  ScanFilter scan_filter_;
  scoped_ptr<DomainRewriteFilter> domain_rewriter_;
  scoped_ptr<UrlLeftTrimFilter> url_trim_filter_;

  typedef std::map<GoogleString, RewriteContext*> PrimaryRewriteContextMap;
  PrimaryRewriteContextMap primary_rewrite_context_map_;

  HtmlResourceSlotSet slots_;
  InlineResourceSlotSet inline_slots_;
  InlineAttributeSlotSet inline_attribute_slots_;
  SrcSetSlotCollectionSet srcset_collections_;

  scoped_ptr<RewriteOptions> options_;

  RewriteDriverPool* controlling_pool_; 

  scoped_ptr<CacheUrlAsyncFetcher::AsyncOpHooks>
      cache_url_async_fetcher_async_op_hooks_;

  UrlSegmentEncoder default_encoder_;

  FilterList early_pre_render_filters_;
  FilterList pre_render_filters_;

  std::vector<ResourceUrlClaimant*> resource_claimants_;

  FilterVector filters_to_delete_;

  QueuedWorkerPool::Sequence* html_worker_;
  QueuedWorkerPool::Sequence* rewrite_worker_;
  QueuedWorkerPool::Sequence* low_priority_rewrite_worker_;
  scoped_ptr<Scheduler::Sequence> scheduler_sequence_;

  Writer* writer_;

  FallbackPropertyPage* fallback_property_page_;

  bool owns_property_page_;

  scoped_ptr<PropertyPage> origin_property_page_;

  UserAgentMatcher::DeviceType device_type_;

  scoped_ptr<CriticalImagesInfo> critical_images_info_;
  scoped_ptr<CriticalSelectorInfo> critical_selector_info_;

  bool xhtml_mimetype_computed_;
  XhtmlStatus xhtml_status_ : 8;

  int num_inline_preview_images_;

  int num_bytes_in_;

  DebugFilter* debug_filter_;

  scoped_ptr<FlushEarlyInfo> flush_early_info_;
  scoped_ptr<DependencyTracker> dependency_tracker_;

  bool can_rewrite_resources_;
  bool is_nested_;

  RequestContextPtr request_context_;

  int64 start_time_ms_;

  scoped_ptr<RequestProperties> request_properties_;

  static int initialized_count_;

  bool defer_instrumentation_script_;

  bool is_amp_;

  AtomicBool executing_rewrite_tasks_;

  DownstreamCachePurger downstream_cache_purger_;

  GoogleString pagespeed_query_params_;

  GoogleString pagespeed_option_cookies_;

  CspContext csp_context_;

  
};

class OptionsAwareHTTPCacheCallback : public HTTPCache::Callback {
 public:
  virtual ~OptionsAwareHTTPCacheCallback();
  virtual bool IsCacheValid(const GoogleString& key,
                            const ResponseHeaders& headers);
  virtual int64 OverrideCacheTtlMs(const GoogleString& key);
  virtual ResponseHeaders::VaryOption RespectVaryOnResources() const;

  static bool IsCacheValid(const GoogleString& key,
                           const RewriteOptions& rewrite_options,
                           const RequestContextPtr& request_ctx,
                           const ResponseHeaders& headers);

 protected:
  OptionsAwareHTTPCacheCallback(
      const RewriteOptions* rewrite_options,
      const RequestContextPtr& request_ctx);

 private:
  const RewriteOptions* rewrite_options_;

  
};

}  

#endif