rewrite_test_base.h

Go to the documentation of this file.

/*
 * Copyright 2010 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http:///www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */



#ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
#define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_

#include <utility>
#include <vector>

#include "net/instaweb/http/public/http_cache.h"
#include "net/instaweb/http/public/http_value.h"
#include "net/instaweb/http/public/logging_proto.h"
#include "net/instaweb/http/public/logging_proto_impl.h"
#include "net/instaweb/http/public/mock_url_fetcher.h"
#include "net/instaweb/http/public/request_context.h"
#include "net/instaweb/rewriter/public/resource.h"
#include "net/instaweb/rewriter/public/resource_namer.h"
#include "net/instaweb/rewriter/public/rewrite_driver.h"
#include "net/instaweb/rewriter/public/rewrite_options.h"
#include "net/instaweb/rewriter/public/server_context.h"
#include "net/instaweb/rewriter/public/test_rewrite_driver_factory.h"
#include "net/instaweb/util/public/mock_property_page.h"
#include "net/instaweb/util/public/property_cache.h"
#include "pagespeed/kernel/base/basictypes.h"
#include "pagespeed/kernel/base/hasher.h"
#include "pagespeed/kernel/base/md5_hasher.h"
#include "pagespeed/kernel/base/mem_file_system.h"
#include "pagespeed/kernel/base/message_handler.h"
#include "pagespeed/kernel/base/mock_hasher.h"
#include "pagespeed/kernel/base/mock_message_handler.h"
#include "pagespeed/kernel/base/mock_timer.h"
#include "pagespeed/kernel/base/scoped_ptr.h"
#include "pagespeed/kernel/base/statistics.h"
#include "pagespeed/kernel/base/string.h"
#include "pagespeed/kernel/base/string_util.h"
#include "pagespeed/kernel/base/timer.h"
#include "pagespeed/kernel/html/html_parse_test_base.h"
#include "pagespeed/kernel/html/html_writer_filter.h"
#include "pagespeed/kernel/http/content_type.h"
#include "pagespeed/kernel/http/http_names.h"
#include "pagespeed/kernel/http/request_headers.h"
#include "pagespeed/kernel/http/response_headers.h"
#include "pagespeed/kernel/http/user_agent_matcher.h"
#include "pagespeed/kernel/util/url_segment_encoder.h"
#include "pagespeed/opt/logging/request_timing_info.h"



namespace net_instaweb {

class AbstractLogRecord;
class CountingUrlAsyncFetcher;
class DelayCache;
class LRUCache;
class MockLogRecord;
class MockScheduler;
class ProcessContext;
class RewriteFilter;
class WaitUrlAsyncFetcher;

class RewriteOptionsTestBase : public HtmlParseTestBaseNoAlloc {
 protected:
  RewriteOptionsTestBase() {
    RewriteOptions::Initialize();
  }
  ~RewriteOptionsTestBase() {
    RewriteOptions::Terminate();
  }
};

class RewriteTestBase : public RewriteOptionsTestBase {
 public:
  static const char kTestData[]; 

  static const char kConfiguredBeaconingKey[];
  static const char kWrongBeaconingKey[];

  enum ActiveServerFlag {
    kPrimary,    
    kSecondary,  
  };

  RewriteTestBase();

  explicit RewriteTestBase(std::pair<TestRewriteDriverFactory*,
                                     TestRewriteDriverFactory*> factories);
  virtual ~RewriteTestBase();

  virtual void SetUp();
  virtual void TearDown();

  virtual bool AddBody() const { return false; }

  virtual TestRewriteDriverFactory* MakeTestFactory();

  void AddRecompressImageFilters();

  void AddFilter(RewriteOptions::Filter filter);

  void AddOtherFilter(RewriteOptions::Filter filter);

  void AddRewriteFilter(RewriteFilter* filter);

  void AddFetchOnlyRewriteFilter(RewriteFilter* filter);

  void AddOtherRewriteFilter(RewriteFilter* filter);

  void SetBaseUrlForFetch(const StringPiece& url);

  void SetDriverRequestHeaders();

  void SetDownstreamCacheDirectives(
    StringPiece downstream_cache_purge_method,
    StringPiece downstream_cache_purge_location_prefix,
    StringPiece rebeaconing_key);

  void SetShouldBeaconHeader(StringPiece rebeaconing_key);

  ResourcePtr CreateResource(const StringPiece& base, const StringPiece& url);

  Timer* timer() { return factory()->mock_timer(); }

  void AppendDefaultHeaders(const ContentType& content_type,
                            GoogleString* text);

  void AppendDefaultHeadersWithCanonical(const ContentType& content_type,
                                         StringPiece canonical_url,
                                         GoogleString* text);

  void ServeResourceFromManyContexts(const GoogleString& resource_url,
                                     const StringPiece& expected_content);

  void ServeResourceFromManyContextsWithUA(
      const GoogleString& resource_url,
      const StringPiece& expected_content,
      const StringPiece& user_agent);

  void ServeResourceFromNewContext(
      const GoogleString& resource_url,
      const StringPiece& expected_content);

  virtual RewriteDriver* html_parse() { return rewrite_driver_; }

  void DefaultResponseHeaders(const ContentType& content_type, int64 ttl_sec,
                              ResponseHeaders* response_headers);

  bool FetchResource(const StringPiece& path, const StringPiece& filter_id,
                     const StringPiece& name, const StringPiece& ext,
                     GoogleString* content);
  bool FetchResource(const StringPiece& path, const StringPiece& filter_id,
                     const StringPiece& name, const StringPiece& ext,
                     GoogleString* content, ResponseHeaders* response);

  bool FetchResourceUrl(const StringPiece& url, GoogleString* content,
                        ResponseHeaders* response);
  bool FetchResourceUrl(const StringPiece& url,
                        RequestHeaders* request_headers,
                        GoogleString* content,
                        ResponseHeaders* response_headers);
  bool FetchResourceUrl(const StringPiece& url, GoogleString* content);

  bool TryFetchResource(const StringPiece& url);

  void SetUseManagedRewriteDrivers(bool use_managed_rewrite_drivers);

  GoogleString CssLinkHref(const StringPiece& url) {
    return StrCat("<link rel=stylesheet href=", url, ">");
  }

  class CssLink {
   public:
    CssLink(const StringPiece& url, const StringPiece& content,
            const StringPiece& media, bool supply_mock);

    class Vector : public std::vector<CssLink*> {
     public:
      ~Vector();
      void Add(const StringPiece& url, const StringPiece& content,
               const StringPiece& media, bool supply_mock);
    };

    bool DecomposeCombinedUrl(StringPiece base_url, GoogleString* base,
                              StringVector* segments, MessageHandler* handler);

    GoogleString url_;
    GoogleString content_;
    GoogleString media_;
    bool supply_mock_;
  };

  void CollectCssLinks(const StringPiece& id, const StringPiece& html,
                       StringVector* css_links);

  void CollectCssLinks(const StringPiece& id, const StringPiece& html,
                       CssLink::Vector* css_links);

  void EncodePathAndLeaf(const StringPiece& filter_id,
                         const StringPiece& hash,
                         const StringVector& name_vector,
                         const StringPiece& ext,
                         ResourceNamer* namer);

  StringVector MultiUrl(const StringPiece& url1) {
    StringVector v;
    v.push_back(url1.as_string());
    return v;
  }

  StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2) {
    StringVector v;
    v.push_back(url1.as_string());
    v.push_back(url2.as_string());
    return v;
  }

  StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2,
                        const StringPiece& url3) {
    StringVector v;
    v.push_back(url1.as_string());
    v.push_back(url2.as_string());
    v.push_back(url3.as_string());
    return v;
  }

  StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2,
                        const StringPiece& url3, const StringPiece& url4) {
    StringVector v;
    v.push_back(url1.as_string());
    v.push_back(url2.as_string());
    v.push_back(url3.as_string());
    v.push_back(url4.as_string());
    return v;
  }

  GoogleString Encode(const StringPiece& path,
                      const StringPiece& filter_id,
                      const StringPiece& hash,
                      const StringPiece& name,
                      const StringPiece& ext) {
    return Encode(path, filter_id, hash, MultiUrl(name), ext);
  }
  GoogleString Encode(const StringPiece& path,
                      const StringPiece& filter_id,
                      const StringPiece& hash,
                      const StringVector& name_vector,
                      const StringPiece& ext);

  GoogleString EncodeNormal(const StringPiece& path,
                            const StringPiece& filter_id,
                            const StringPiece& hash,
                            const StringPiece& name,
                            const StringPiece& ext) {
    return EncodeNormal(path, filter_id, hash, MultiUrl(name), ext);
  }
  GoogleString EncodeNormal(const StringPiece& path,
                            const StringPiece& filter_id,
                            const StringPiece& hash,
                            const StringVector& name_vector,
                            const StringPiece& ext);

  GoogleString EncodeWithBase(const StringPiece& base,
                              const StringPiece& path,
                              const StringPiece& filter_id,
                              const StringPiece& hash,
                              const StringPiece& name,
                              const StringPiece& ext) {
    return EncodeWithBase(base, path, filter_id, hash, MultiUrl(name), ext);
  }
  GoogleString EncodeWithBase(const StringPiece& base,
                              const StringPiece& path,
                              const StringPiece& filter_id,
                              const StringPiece& hash,
                              const StringVector& name_vector,
                              const StringPiece& ext);

  GoogleString EncodeImage(int width, int height,
                           StringPiece filename, StringPiece hash,
                           StringPiece rewritten_ext);

  GoogleString AddOptionsToEncodedUrl(const StringPiece& url,
                                      const StringPiece& options);

  static GoogleString ChangeSuffix(
      StringPiece old_url, bool append_new_suffix,
      StringPiece old_suffix, StringPiece new_suffix);

  void SetupWaitFetcher();
  void CallFetcherCallbacks();
  void OtherCallFetcherCallbacks();
  RewriteOptions* options() const { return options_; }
  RewriteOptions* other_options() const { return other_options_; }

  void SetRewriteOptions(RewriteOptions* opts);

  bool AddDomain(StringPiece domain);

  bool AddOriginDomainMapping(StringPiece to_domain, StringPiece from_domain);

  bool AddRewriteDomainMapping(StringPiece to_domain, StringPiece from_domain);

  bool AddShard(StringPiece domain, StringPiece shards);

  void TestServeFiles(const ContentType* content_type,
                      const StringPiece& filter_id,
                      const StringPiece& rewritten_ext,
                      const StringPiece& orig_name,
                      const StringPiece& orig_content,
                      const StringPiece& rewritten_name,
                      const StringPiece& rewritten_content);

  void ValidateFallbackHeaderSanitization(StringPiece filter_id);

  TestRewriteDriverFactory* factory() { return factory_.get(); }
  TestRewriteDriverFactory* other_factory() { return other_factory_.get(); }

  void UseMd5Hasher() {
    server_context_->set_hasher(&md5_hasher_);
    server_context_->http_cache()->set_hasher(&md5_hasher_);
    other_server_context_->set_hasher(&md5_hasher_);
    other_server_context_->http_cache()->set_hasher(&md5_hasher_);
  }


  void SetDefaultLongCacheHeaders(const ContentType* content_type,
                                  ResponseHeaders* header) {
    server_context_->SetDefaultLongCacheHeaders(
        content_type, StringPiece(), StringPiece(), header);
  }

  void SetFetchResponse(const StringPiece& url,
                        const ResponseHeaders& response_header,
                        const StringPiece& response_body) {
    mock_url_fetcher()->SetResponse(url, response_header, response_body);
  }

  void SetResponseWithDefaultHeaders(const StringPiece& relative_url,
                                     const ContentType& content_type,
                                     const StringPiece& content,
                                     int64 ttl_sec);

  bool LoadFile(const StringPiece& filename, GoogleString* contents);

  void AddFileToMockFetcher(const StringPiece& url,
                            const StringPiece& filename,
                            const ContentType& content_type, int64 ttl_sec);

  void AddToResponse(const StringPiece& url,
                     const StringPiece& name,
                     const StringPiece& value) {
    mock_url_fetcher()->AddToResponse(url, name, value);
  }

  void SetFetchResponse404(const StringPiece& url);

  void SetFetchFailOnUnexpected(bool fail) {
    mock_url_fetcher()->set_fail_on_unexpected(fail);
  }
  void FetcherUpdateDateHeaders() {
    mock_url_fetcher()->set_timer(timer());
    mock_url_fetcher()->set_update_date_headers(true);
  }
  void ClearFetcherResponses() { mock_url_fetcher()->Clear(); }

  virtual void ClearStats();

  void ClearRewriteDriver();

  MockUrlFetcher* mock_url_fetcher() {
    return &mock_url_fetcher_;
  }
  Hasher* hasher() { return server_context_->hasher(); }
  DelayCache* delay_cache() { return factory_->delay_cache(); }
  LRUCache* lru_cache() { return factory_->lru_cache(); }
  Statistics* statistics() { return factory_->statistics(); }
  MemFileSystem* file_system() { return factory_->mem_file_system(); }
  HTTPCache* http_cache() { return server_context_->http_cache(); }
  PropertyCache* page_property_cache() {
    return server_context_->page_property_cache();
  }
  MockMessageHandler* message_handler() {
    return factory_->mock_message_handler();
  }

  RewriteDriver* rewrite_driver() { return rewrite_driver_; }
  RewriteDriver* other_rewrite_driver() { return other_rewrite_driver_; }

  MockScheduler* mock_scheduler() { return factory_->mock_scheduler(); }

  int64 start_time_ms() const { return factory_->kStartTimeMs; }

  bool ReadFile(const char* filename, GoogleString* contents) {
    return file_system()->ReadFile(filename, contents, message_handler());
  }
  bool WriteFile(const char* filename, const StringPiece& contents) {
    return file_system()->WriteFile(filename, contents, message_handler());
  }

  ServerContext* server_context() { return server_context_; }
  ServerContext* other_server_context() { return other_server_context_; }
  CountingUrlAsyncFetcher* counting_url_async_fetcher() {
    return factory_->counting_url_async_fetcher();
  }
  void SetMockHashValue(const GoogleString& value) {
    factory_->mock_hasher()->set_hash_value(value);
  }

  void SetCacheDelayUs(int64 delay_us);

  void SetupWriter() override;

  RewriteDriver* MakeDriver(ServerContext* server_context,
                            RewriteOptions* options);

  GoogleString AbsolutifyUrl(const StringPiece& in);

  void TestRetainExtraHeaders(const StringPiece& name,
                              const StringPiece& filter_id,
                              const StringPiece& ext);

  const UrlSegmentEncoder* FindEncoder(const StringPiece& id) const;

  void SetUseTestUrlNamer(bool use_test_url_namer);

  GoogleString EncodeCssName(const StringPiece& name,
                             bool supports_webp,
                             bool can_inline);

  bool ReadIfCached(const ResourcePtr& resource);

  void InitiateResourceRead(const ResourcePtr& resource);

  HTTPCache::FindResult HttpBlockingFind(
      const GoogleString& key, HTTPCache* http_cache, HTTPValue* value_out,
      ResponseHeaders* headers);

  HTTPCache::FindResult HttpBlockingFindStatus(
      const GoogleString& key, HTTPCache* http_cache);

  HTTPCache::FindResult HttpBlockingFindWithOptions(
      const RewriteOptions* options,
      const GoogleString& key, HTTPCache* http_cache, HTTPValue* value_out,
      ResponseHeaders* headers);

  void SetXhtmlMimetype() { SetMimetype("application/xhtml+xml"); }

  void SetHtmlMimetype() { SetMimetype("text/html"); }

  void SetMimetype(const StringPiece& mimetype);

  void CheckFetchFromHttpCache(
      StringPiece url,
      StringPiece expected_contents,
      int64 expected_expiration_ms);

  const PropertyCache::Cohort*  SetupCohort(
      PropertyCache* cache, const GoogleString& cohort) {
    return factory()->SetupCohort(cache, cohort);
  }

  void SetupSharedCache();

  MockPropertyPage* NewMockPage(const StringPiece& url,
                                const StringPiece& options_signature_hash,
                                UserAgentMatcher::DeviceType device_type) {
    return new MockPropertyPage(
        server_context_->thread_system(),
        server_context_->page_property_cache(),
        url,
        options_signature_hash,
        UserAgentMatcher::DeviceTypeSuffix(device_type));
  }

  MockPropertyPage* NewMockPage(const StringPiece& url) {
    return NewMockPage(url, "hash", UserAgentMatcher::kDesktop);
  }

  void SetMockLogRecord();

  MockLogRecord* mock_log_record();

  GoogleString GetLazyloadScriptHtml();
  GoogleString GetLazyloadPostscriptHtml();

  void SetCacheInvalidationTimestamp();

  void SetCacheInvalidationTimestampForUrl(
      StringPiece url, bool ignores_metadata_and_pcache);

  void EnableCachePurge();

  void EnableDebug();

  void DebugWithMessage(StringPiece expected_debug_message) {
    EnableDebug();

    expected_debug_message.CopyToString(&debug_message_);
  }

  GoogleString DebugMessage(StringPiece url);

  static const ProcessContext& process_context();

  void DisableGzip();

  bool WasGzipped(const ResponseHeaders& response_headers);

 protected:
  const HTTPCache::FindResult kFoundResult;
  const HTTPCache::FindResult kNotFoundResult;

  void Init();

  virtual RequestContextPtr CreateRequestContext();

  void CallFetcherCallbacksForDriver(WaitUrlAsyncFetcher* fetcher,
                                     RewriteDriver* driver);

  void PopulateDefaultHeaders(const ContentType& content_type,
                              int64 original_content_length,
                              ResponseHeaders* headers);

  void SetActiveServer(ActiveServerFlag server_to_use);

  void AdvanceTimeUs(int64 delay_ms);
  void AdvanceTimeMs(int64 delay_ms) { AdvanceTimeUs(delay_ms * Timer::kMsUs); }
  void SetTimeUs(int64 time_us);
  void SetTimeMs(int64 time_ms) { SetTimeUs(time_ms * Timer::kMsUs); }

  void AdjustTimeUsWithoutWakingAlarms(int64 time_us);

  const RequestTimingInfo& timing_info();
  RequestTimingInfo* mutable_timing_info();

  virtual RequestContextPtr request_context();

  LoggingInfo* logging_info();

  const MetadataCacheInfo& metadata_cache_info() {
    return logging_info()->metadata_cache_info();
  }

  GoogleString AppliedRewriterStringFromLog();

  void VerifyRewriterInfoEntry(AbstractLogRecord* log_record,
                               const GoogleString& id,
                               int url_index,
                               int rewriter_info_index,
                               int rewriter_info_size,
                               int url_list_size,
                               const GoogleString& url);

  void SetCurrentUserAgent(const StringPiece& user_agent) {
    current_user_agent_ = user_agent;
  }

  void SetupForWebp() {
    SetCurrentUserAgent("webp");
    AddRequestAttribute(HttpAttributes::kAccept, "image/webp");
  }

  void SetupForWebpLossless() {
    SetCurrentUserAgent("webp-la");
    AddRequestAttribute(HttpAttributes::kAccept, "image/webp");
  }

  void SetupForWebpAnimated() {
    SetCurrentUserAgent("webp-animated");
    AddRequestAttribute(HttpAttributes::kAccept, "image/webp");
  }

  void AddRequestAttribute(StringPiece name, StringPiece value);

  void PopulateRequestHeaders(RequestHeaders* request_headers);

  virtual void ParseUrl(StringPiece url, StringPiece html_input);

  GoogleString ExpectedNonce();

  GoogleString HttpCacheKey(StringPiece url) {
    return http_cache()->CompositeKey(url, rewrite_driver_->CacheFragment());
  }

  int TimedValue(StringPiece name);

  MockUrlFetcher mock_url_fetcher_;
  scoped_ptr<Statistics> statistics_;

  scoped_ptr<TestRewriteDriverFactory> factory_;
  scoped_ptr<TestRewriteDriverFactory> other_factory_;
  ServerContext* server_context_;
  RewriteDriver* rewrite_driver_;
  ServerContext* other_server_context_;
  RewriteDriver* other_rewrite_driver_;
  scoped_ptr<HtmlWriterFilter> other_html_writer_filter_;
  ActiveServerFlag active_server_;
  bool use_managed_rewrite_drivers_;
  StringPiece current_user_agent_;
  StringVector request_attribute_names_;
  StringVector request_attribute_values_;

  MD5Hasher md5_hasher_;

  RewriteOptions* options_; 
  RewriteOptions* other_options_; 
  UrlSegmentEncoder default_encoder_;
  ResponseHeaders response_headers_;
  const GoogleString kEtag0; 
  uint64 expected_nonce_;

  GoogleString debug_message_; 

 private:
  void ValidateFallbackHeaderSanitizationHelper(
      StringPiece filter_id, StringPiece origin_content_type, bool expect_load);
};

}  

#endif