Page Speed Optimization Libraries  1.11.33.2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Pages
rewrite_context.h
Go to the documentation of this file.
1 /*
2  * Copyright 2011 Google Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http:///www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
18 
19 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_CONTEXT_H_
20 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_CONTEXT_H_
21 
22 #include <set>
23 #include <vector>
24 
26 #include "net/instaweb/rewriter/cached_result.pb.h"
32 #include "pagespeed/controller/schedule_rewrite_callback.h"
33 #include "pagespeed/kernel/base/basictypes.h"
34 #include "pagespeed/kernel/base/function.h"
35 #include "pagespeed/kernel/base/scoped_ptr.h"
36 #include "pagespeed/kernel/base/string.h"
37 #include "pagespeed/kernel/base/string_util.h"
38 #include "pagespeed/kernel/http/google_url.h"
39 #include "pagespeed/kernel/http/response_headers.h"
40 #include "pagespeed/kernel/util/url_segment_encoder.h"
41 
42 namespace net_instaweb {
43 
44 class AsyncFetch;
45 class MessageHandler;
46 class NamedLock;
47 class RequestTrace;
48 class RewriteDriver;
49 class RewriteOptions;
50 class Statistics;
51 class Variable;
52 class FreshenMetadataUpdateManager;
53 
147  public:
148  typedef std::vector<InputInfo*> InputInfoStarVector;
149  static const char kNumRewritesAbandonedForLockContention[];
150  static const char kNumDeadlineAlarmInvocations[];
151  static const char kNumDistributedRewriteSuccesses[];
152  static const char kNumDistributedRewriteFailures[];
153  static const char kNumDistributedMetadataFailures[];
155  static const char kDistributedExt[];
157  static const char kDistributedHash[];
158  static const char kHashMismatchMessage[];
159 
164  : cache_ok(false),
165  can_revalidate(false),
166  useable_cache_content(false),
167  is_stale_rewrite(false),
168  partitions(new OutputPartitions) {}
169 
170  bool cache_ok;
171  bool can_revalidate;
172  bool useable_cache_content;
173  bool is_stale_rewrite;
174  InputInfoStarVector revalidate;
175  scoped_ptr<OutputPartitions> partitions;
176  };
177 
180  public:
182  virtual ~CacheLookupResultCallback();
183  virtual void Done(const GoogleString& cache_key,
184  CacheLookupResult* result) = 0;
185  private:
186 
187  };
188 
191  RewriteContext(RewriteDriver* driver,
193  ResourceContext* resource_context);
194  virtual ~RewriteContext();
195 
200  int num_slots() const { return slots_.size(); }
201  ResourceSlotPtr slot(int index) const { return slots_[index]; }
202 
205  int num_outputs() const { return outputs_.size(); }
206  OutputResourcePtr output(int i) const { return outputs_[i]; }
207 
210  int num_output_partitions() const;
211  const CachedResult* output_partition(int i) const;
212  CachedResult* output_partition(int i);
213 
216  bool chained() const { return chained_; }
217 
221  void AddSlot(const ResourceSlotPtr& slot);
222 
225  void RemoveLastSlot();
226 
232  void AddNestedContext(RewriteContext* context);
233 
234  void CallFetchInputs();
235  void CallLockFailed();
236  void CallStartFetchImpl();
237 
248  void Initiate();
249 
255  bool Fetch(const OutputResourcePtr& output_resource,
256  AsyncFetch* fetch,
257  MessageHandler* message_handler);
258 
261  bool slow() const { return slow_; }
262 
264  bool is_metadata_cache_miss() const { return is_metadata_cache_miss_; }
265 
267  bool has_parent() const { return parent_ != NULL; }
268 
271  bool IsNestedIn(StringPiece id) const;
272 
274  RewriteContext* parent() { return parent_; }
275  const RewriteContext* parent() const { return parent_; }
276 
278  int num_nested() const { return nested_.size(); }
279  RewriteContext* nested(int i) const { return nested_[i]; }
280 
281  RewriteDriver* Driver() const {
282  return driver_;
283  }
284 
286  void set_force_rewrite(bool x) { force_rewrite_ = x; }
287 
288  bool rewrite_uncacheable() const { return rewrite_uncacheable_; }
289  void set_rewrite_uncacheable(bool rewrite_uncacheable) {
290  rewrite_uncacheable_ = rewrite_uncacheable;
291  }
292 
293  const ResourceContext* resource_context() const {
294  return resource_context_.get();
295  }
296 
298  GoogleString ToString() const;
299  GoogleString ToStringWithPrefix(StringPiece prefix) const;
300 
302  static void InitStats(Statistics* stats);
303 
304  protected:
305  typedef std::vector<GoogleUrl*> GoogleUrlStarVector;
306 
311 
315  ServerContext* FindServerContext() const;
316  const RewriteOptions* Options() const;
317 
318  OutputPartitions* partitions() { return partitions_.get(); }
319 
322  void AddRecheckDependency();
323 
327  virtual bool OptimizationOnly() const { return true; }
328 
346  virtual bool Partition(OutputPartitions* partitions,
347  OutputResourceVector* outputs);
348 
354  virtual void PartitionAsync(OutputPartitions* partitions,
355  OutputResourceVector* outputs);
356 
360  void PartitionDone(RewriteResult result);
361 
365 
384  virtual void Rewrite(int partition_index,
385  CachedResult* partition,
386  const OutputResourcePtr& output) = 0;
387 
392  void RewriteDone(RewriteResult result, int partition_index);
393 
403  virtual bool SendFallbackResponse(StringPiece output_url_base,
404  StringPiece contents,
406  MessageHandler* handler);
407 
411  void StartNestedTasks();
412 
417  virtual void Harvest();
418 
430  virtual void Render();
431 
442  virtual void WillNotRender();
443 
447  virtual void Cancel();
448 
451 
458  virtual const UrlSegmentEncoder* encoder() const;
459 
462  virtual GoogleString CacheKeySuffix() const;
463 
470  virtual GoogleString UserAgentCacheKey(
471  const ResourceContext* context) const {
472  return "";
473  }
474 
479  virtual void EncodeUserAgentIntoResourceContext(ResourceContext* context) {}
480 
482  virtual const char* id() const = 0;
483 
492  virtual OutputResourceKind kind() const = 0;
493 
497 
500  void AttachDependentRequestTrace(const StringPiece& label);
501 
505  RequestTrace* dependent_request_trace() { return dependent_request_trace_; }
506 
509  void TracePrintf(const char* fmt, ...);
510 
518 
528  virtual void StartFetchReconstruction();
529 
535  bool ShouldDistributeRewrite() const;
536 
539  bool IsDistributedRewriteForHtml() const;
540 
544  void DistributeRewrite();
545 
551  void DetachFetch();
552 
557  virtual bool DecodeFetchUrls(const OutputResourcePtr& output_resource,
558  MessageHandler* message_handler,
559  GoogleUrlStarVector* url_vector);
560 
565  virtual void FixFetchFallbackHeaders(const CachedResult& cached_result,
566  ResponseHeaders* headers);
567 
570  virtual void FetchCallbackDone(bool success);
571 
576  virtual void FetchTryFallback(const GoogleString& url,
577  const StringPiece& hash);
578 
580  void Freshen();
581 
582  bool notify_driver_on_fetch_done() const {
583  return notify_driver_on_fetch_done_;
584  }
585  void set_notify_driver_on_fetch_done(bool value) {
586  notify_driver_on_fetch_done_ = value;
587  }
588 
593  bool block_distribute_rewrite() const { return block_distribute_rewrite_; }
594  void set_block_distribute_rewrite(const bool x) {
595  block_distribute_rewrite_ = x;
596  }
597 
599  AsyncFetch* async_fetch();
600 
602  bool FetchContextDetached();
603 
605  MessageHandler* fetch_message_handler();
606 
608  bool stale_rewrite() const { return stale_rewrite_; }
609 
613  virtual int64 GetRewriteDeadlineAlarmMs() const;
614 
616  virtual bool CreationLockBeforeStartFetch() const;
617 
625  virtual bool FailOnHashMismatch() const { return false; }
626 
630  virtual bool ScheduleViaCentralController() { return false; }
631 
634  void ObtainLockForCreation(ServerContext* server_context, Function* callback);
635 
640  void ReleaseCreationLock(bool succeeded);
641 
646  OutputResourcePtr output_resource,
647  const GoogleUrl& gurl,
648  RewriteContext* rewrite_context,
649  RewriteDriver* driver,
650  GoogleString* error_out,
651  CacheLookupResultCallback* callback);
652 
653  private:
654  class DistributedRewriteCallback;
655  class DistributedRewriteFetch;
656  class OutputCacheCallback;
657  class WriteIfChanged;
658  class LookupMetadataForOutputResourceCallback;
659  class HTTPCacheCallback;
660  class ResourceCallbackUtils;
661  class ResourceFetchCallback;
662  class ResourceReconstructCallback;
663  class ResourceRevalidateCallback;
664  class InvokeRewriteFunction;
665  class RewriteFreshenCallback;
666  class TryLockFunction;
667  friend class RewriteDriver;
668 
669  typedef std::set<RewriteContext*> ContextSet;
670 
674  enum FallbackCondition {
675  kFallbackDiscretional,
676  kFallbackEmergency
678  };
680 
682  void Start();
683  void SetPartitionKey();
684  void StartFetch();
685  void StartFetchImpl();
686  void CancelFetch();
687  void OutputCacheDone(CacheLookupResult* cache_result);
688  void OutputCacheHit(bool write_partitions);
689  void OutputCacheRevalidate(const InputInfoStarVector& to_revalidate);
690  void OutputCacheMiss();
691  void ResourceFetchDone(bool success, ResourcePtr resource, int slot_index);
692  void ResourceRevalidateDone(InputInfo* input_info, bool success);
693  void LogMetadataCacheInfo(bool cache_ok, bool can_revalidate);
694 
699  void RepeatedSuccess(const RewriteContext* primary);
700  void RepeatedFailure();
701 
710  void Finalize();
711 
713  NamedLock* Lock();
714 
717  GoogleString LockName() const;
718 
728  void FetchInputs();
729 
731  void LockFailed();
732 
737  void DistributeRewriteDone(bool success);
738 
742  bool ParseAndRemoveMetadataFromResponseHeaders(
743  ResponseHeaders* response_headers, CacheLookupResult* cache_result);
744 
747  bool CreateOutputResourceFromContent(const CachedResult& cached_result,
748  const ResponseHeaders& response_headers,
749  StringPiece content,
750  OutputResourcePtr* output_resource);
751 
761  GoogleString DistributedFetchUrl(StringPiece url);
762 
765  bool IsFetchRewrite() const { return fetch_.get() != NULL; }
766 
771  void NestedRewriteDone(const RewriteContext* context);
772 
776  void Activate();
777 
801  void Propagate(bool render_slots);
802 
809  void StartRewriteForHtml();
810  void StartRewriteForFetch();
811 
818  bool ReadyToRewrite() const;
819 
824  void DetachSlots();
825 
828  void RunSuccessors();
829 
832  void WritePartition();
833 
843  void FinalizeRewriteForHtml();
844 
850  void RetireRewriteForHtml(bool permit_render);
851 
854  void MarkSlow();
855 
858  void MarkTooBusy();
859 
863  void CollectDependentTopLevel(ContextSet* contexts);
864 
867  void RewriteDoneImpl(RewriteResult result, int partition_index);
868 
871  void StartNestedTasksImpl();
872 
876  void RenderPartitionOnDetach(int partition_index);
877 
880  bool PrepareFetch(
881  const OutputResourcePtr& output_resource,
882  AsyncFetch* fetch,
883  MessageHandler* message_handler);
884 
887  bool CreateOutputResourceForCachedOutput(const CachedResult* cached_result,
888  OutputResourcePtr* output_resource);
889 
891  void FetchCacheDone(CacheLookupResult* cache_result);
892 
895  void FetchFallbackCacheDone(HTTPCache::FindResult result,
896  HTTPCache::Callback* data);
897 
902  bool CanFetchFallbackToOriginal(FallbackCondition circumstance) const;
903 
906  bool HasDuplicateOtherDependency(const InputInfo& input);
907 
911  void CheckAndAddOtherDependency(const InputInfo& input);
912 
915  void CheckAndFreshenResource(const InputInfo& input_info,
916  ResourcePtr resource, int partition_index,
917  int input_index,
918  FreshenMetadataUpdateManager* freshen_manager);
919  ResourcePtr CreateUrlResource(const StringPiece& input_url);
920 
922  ResourceSlotVector slots_;
923 
927  std::vector<bool> render_slots_;
928 
939 
940  bool started_;
941  scoped_ptr<OutputPartitions> partitions_;
942  OutputResourceVector outputs_;
943  int outstanding_fetches_;
944  int outstanding_rewrites_;
945  scoped_ptr<ResourceContext> resource_context_;
946  GoogleString partition_key_;
947 
948  UrlSegmentEncoder default_encoder_;
949 
952  scoped_ptr<NamedLock> lock_;
953 
957  class FetchContext;
958  scoped_ptr<FetchContext> fetch_;
959 
962  std::vector<RewriteContext*> successors_;
963 
967  std::vector<RewriteContext*> repeated_;
968 
972  int num_pending_nested_;
973  std::vector<RewriteContext*> nested_;
974 
976  RewriteContext* parent_;
977 
986  RewriteDriver* driver_;
987 
989  int num_predecessors_;
990 
993  bool chained_;
994 
1012 
1014  bool rewrite_done_;
1015 
1027  bool ok_to_write_output_partitions_;
1028 
1032  bool was_too_busy_;
1033 
1037  bool slow_;
1038 
1040  bool revalidate_ok_;
1041 
1044  bool notify_driver_on_fetch_done_;
1045 
1048  bool force_rewrite_;
1049 
1052  bool stale_rewrite_;
1053 
1056  bool is_metadata_cache_miss_;
1057 
1060  bool rewrite_uncacheable_;
1061 
1064  RequestTrace* dependent_request_trace_;
1065 
1068  bool block_distribute_rewrite_;
1069 
1071  scoped_ptr<DistributedRewriteFetch> distributed_fetch_;
1072 
1074  StringIntMap other_dependency_map_;
1075 
1079  scoped_ptr<ScheduleRewriteContext> schedule_rewrite_context_;
1080 
1081  Variable* const num_rewrites_abandoned_for_lock_contention_;
1082  Variable* const num_distributed_rewrite_failures_;
1083  Variable* const num_distributed_rewrite_successes_;
1084  Variable* const num_distributed_metadata_failures_;
1085 
1086 };
1087 
1088 }
1089 
1090 #endif
RewriteContext * parent()
Allows a nested rewriter to walk up its parent hierarchy.
Definition: rewrite_context.h:274
static const char kDistributedExt[]
The extension used for all distributed fetch URLs.
Definition: rewrite_context.h:155
virtual bool Partition(OutputPartitions *partitions, OutputResourceVector *outputs)
bool stale_rewrite() const
Indicates whether we are serving a stale rewrite.
Definition: rewrite_context.h:608
bool IsDistributedRewriteForHtml() const
bool block_distribute_rewrite() const
Definition: rewrite_context.h:593
int num_outputs() const
Definition: rewrite_context.h:205
static bool LookupMetadataForOutputResourceImpl(OutputResourcePtr output_resource, const GoogleUrl &gurl, RewriteContext *rewrite_context, RewriteDriver *driver, GoogleString *error_out, CacheLookupResultCallback *callback)
void AddNestedContext(RewriteContext *context)
void ReleaseCreationLock(bool succeeded)
virtual OutputResourceKind kind() const =0
void ObtainLockForCreation(ServerContext *server_context, Function *callback)
virtual bool ScheduleViaCentralController()
Definition: rewrite_context.h:630
static void InitStats(Statistics *stats)
Initializes statistics.
virtual const UrlSegmentEncoder * encoder() const
Definition: rewrite_context.h:162
virtual GoogleString UserAgentCacheKey(const ResourceContext *context) const
Definition: rewrite_context.h:470
void Freshen()
Freshens resources proactively to avoid expiration in the near future.
int num_output_partitions() const
void TracePrintf(const char *fmt,...)
virtual bool DecodeFetchUrls(const OutputResourcePtr &output_resource, MessageHandler *message_handler, GoogleUrlStarVector *url_vector)
bool ShouldDistributeRewrite() const
bool IsNestedIn(StringPiece id) const
virtual bool FailOnHashMismatch() const
Definition: rewrite_context.h:625
void RewriteDone(RewriteResult result, int partition_index)
void PartitionDone(RewriteResult result)
bool Fetch(const OutputResourcePtr &output_resource, AsyncFetch *fetch, MessageHandler *message_handler)
ServerContext * FindServerContext() const
void CrossThreadPartitionDone(RewriteResult result)
Used for LookupMetadataForOutputResource.
Definition: rewrite_context.h:179
virtual const char * id() const =0
Returns the filter ID.
virtual bool OptimizationOnly() const
Definition: rewrite_context.h:327
Definition: async_fetch.h:53
virtual void StartFetchReconstruction()
int num_nested() const
Accessors for the nested rewrites.
Definition: rewrite_context.h:278
virtual void FixFetchFallbackHeaders(const CachedResult &cached_result, ResponseHeaders *headers)
RequestTrace * dependent_request_trace()
Definition: rewrite_context.h:505
Definition: rewrite_driver.h:98
virtual bool SendFallbackResponse(StringPiece output_url_base, StringPiece contents, AsyncFetch *async_fetch, MessageHandler *handler)
Definition: server_context.h:100
RewriteResult
Definition: rewrite_result.h:27
virtual GoogleString CacheKeySuffix() const
bool chained() const
Definition: rewrite_context.h:216
Definition: rewrite_context.h:146
int num_slots() const
Definition: rewrite_context.h:200
virtual bool CreationLockBeforeStartFetch() const
Should the context call LockForCreation before checking the cache?
virtual void Rewrite(int partition_index, CachedResult *partition, const OutputResourcePtr &output)=0
virtual void FetchCallbackDone(bool success)
virtual void EncodeUserAgentIntoResourceContext(ResourceContext *context)
Definition: rewrite_context.h:479
GoogleString ToString() const
Returns debug information about this RewriteContext.
static const char kDistributedHash[]
The hash value used for all distributed fetch URLs.
Definition: rewrite_context.h:157
virtual int64 GetRewriteDeadlineAlarmMs() const
bool slow() const
Definition: rewrite_context.h:261
bool is_metadata_cache_miss() const
This particular rewrite was a metadata cache miss.
Definition: rewrite_context.h:264
MessageHandler * fetch_message_handler()
The message handler for the fetch.
virtual void PartitionAsync(OutputPartitions *partitions, OutputResourceVector *outputs)
bool FetchContextDetached()
Is fetch_ detached? Only call this in the fetch flow.
RewriteContext(RewriteDriver *driver, RewriteContext *parent, ResourceContext *resource_context)
void AddSlot(const ResourceSlotPtr &slot)
virtual void FetchTryFallback(const GoogleString &url, const StringPiece &hash)
bool has_parent() const
Returns true if this is a nested rewriter.
Definition: rewrite_context.h:267
void set_force_rewrite(bool x)
If called with true, forces a rewrite and re-generates the output.
Definition: rewrite_context.h:286
void AttachDependentRequestTrace(const StringPiece &label)
OutputResourceKind
Definition: output_resource_kind.h:26
AsyncFetch * async_fetch()
Note that the following must only be called in the fetch flow.