19 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
20 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
26 #include "base/logging.h"
29 #include "net/instaweb/http/public/request_context.h"
31 #include "net/instaweb/rewriter/cached_result.pb.h"
36 #include "net/instaweb/rewriter/public/inline_resource_slot.h"
46 #include "pagespeed/kernel/base/abstract_mutex.h"
47 #include "pagespeed/kernel/base/atomic_bool.h"
48 #include "pagespeed/kernel/base/basictypes.h"
49 #include "pagespeed/kernel/base/function.h"
50 #include "pagespeed/kernel/base/printf_format.h"
51 #include "pagespeed/kernel/base/proto_util.h"
52 #include "pagespeed/kernel/base/scoped_ptr.h"
53 #include "pagespeed/kernel/base/string.h"
54 #include "pagespeed/kernel/base/string_util.h"
55 #include "pagespeed/kernel/base/thread_annotations.h"
56 #include "pagespeed/kernel/base/thread_system.h"
57 #include "pagespeed/kernel/base/writer.h"
58 #include "pagespeed/kernel/html/html_element.h"
59 #include "pagespeed/kernel/html/html_filter.h"
60 #include "pagespeed/kernel/html/html_node.h"
61 #include "pagespeed/kernel/html/html_parse.h"
62 #include "pagespeed/kernel/http/content_type.h"
63 #include "pagespeed/kernel/http/google_url.h"
64 #include "pagespeed/kernel/http/request_headers.h"
65 #include "pagespeed/kernel/http/response_headers.h"
66 #include "pagespeed/kernel/http/user_agent_matcher.h"
67 #include "pagespeed/kernel/thread/queued_worker_pool.h"
68 #include "pagespeed/kernel/thread/scheduler.h"
69 #include "pagespeed/kernel/thread/sequence.h"
70 #include "pagespeed/kernel/util/categorized_refcount.h"
71 #include "pagespeed/kernel/util/url_segment_encoder.h"
72 #include "pagespeed/opt/http/property_cache.h"
74 namespace net_instaweb {
76 class AbstractLogRecord;
79 class DependencyTracker;
81 class DomainRewriteFilter;
82 class FallbackPropertyPage;
85 class FlushEarlyRenderInfo;
86 class HtmlWriterFilter;
88 class RequestProperties;
90 class RewriteDriverPool;
93 class UrlLeftTrimFilter;
136 kInlineUnauthorizedResources,
137 kInlineOnlyAuthorizedResources
142 kIntendedForInlining,
168 FileSystem* file_system,
193 static void InitStats(Statistics* statistics);
197 static void Terminate();
209 bool MayCacheExtendImages()
const;
210 bool MayCacheExtendPdfs()
const;
211 bool MayCacheExtendScripts()
const;
213 const GoogleString& user_agent()
const {
return user_agent_; }
216 return request_properties_.get();
222 bool write_property_cache_dom_cohort()
const {
223 return write_property_cache_dom_cohort_;
225 void set_write_property_cache_dom_cohort(
bool x) {
226 write_property_cache_dom_cohort_ = x;
229 RequestContextPtr request_context() {
return request_context_; }
230 void set_request_context(
const RequestContextPtr& x);
239 void TraceLiteral(
const char* literal);
240 void TraceString(
const GoogleString& s);
245 return flush_occurred_ ? NULL : response_headers_;
255 return response_headers_;
262 response_headers_ = headers;
275 const RequestHeaders* request_headers()
const {
276 return request_headers_.get();
279 UserAgentMatcher* user_agent_matcher()
const {
280 DCHECK(server_context() != NULL);
281 return server_context()->user_agent_matcher();
340 Writer* writer()
const {
return writer_; }
370 bool FetchResource(
const StringPiece& url, AsyncFetch* fetch);
390 AsyncFetch* async_fetch);
399 RewriteFilter* filter,
400 AsyncFetch* async_fetch);
409 RewriteFilter** filter)
const;
417 const RewriteOptions* options_to_use,
418 const UrlNamer* url_namer,
419 ResourceNamer* name_out,
421 RewriteFilter** filter_out)
const;
436 GoogleString* error_out,
437 RewriteContext::CacheLookupResultCallback* callback);
441 StringVector* decoded_urls)
const;
447 const UrlNamer* url_namer,
448 StringVector* decoded_urls)
const;
450 FileSystem* file_system() {
return file_system_; }
451 UrlAsyncFetcher* async_fetcher() {
return url_async_fetcher_; }
459 UrlAsyncFetcher* distributed_fetcher() {
return distributed_async_fetcher_; }
462 distributed_async_fetcher_ = fetcher;
471 ServerContext* server_context()
const {
return server_context_; }
472 Statistics* statistics()
const;
482 controlling_pool_ = pool;
483 options_.reset(options);
495 virtual bool StartParseId(
const StringPiece& url,
const StringPiece&
id,
496 const ContentType& content_type);
515 const char* msg, ...) INSTAWEB_PRINTF_FORMAT(3, 4);
519 const UrlSegmentEncoder* encoder,
520 const ResourceContext* data,
521 const ResourcePtr& input_resource,
523 GoogleUrl* mapped_gurl,
524 GoogleString* failure_reason);
538 const
char* filter_id,
539 const UrlSegmentEncoder* encoder,
540 const ResourceContext* data,
541 const ResourcePtr& input_resource,
543 GoogleString* failure_reason);
562 const StringPiece& mapped_path, const StringPiece& unmapped_path,
563 const StringPiece&
base_url, const StringPiece& filter_id,
565 GoogleString* failure_reason);
570 const StringPiece& filter_id,
571 const StringPiece& name,
578 const GoogleUrl& unmapped_gurl, const StringPiece& filter_id,
580 GoogleString* failure_reason);
585 const StringPiece& mapped_path, const StringPiece& unmapped_path,
586 const StringPiece& filter_id, const StringPiece& name,
589 decoded_base_url_.AllExceptLeaf(),
590 filter_id, name, kind, failure_reason);
596 const StringPiece& path,
const StringPiece& filter_id,
598 GoogleString* failure_reason) {
600 kind, failure_reason);
609 bool* is_authorized);
627 const GoogleUrl& input_url,
630 bool* is_authorized);
636 const StringPiece& absolute_url);
655 const GoogleUrl& input_url,
658 bool* is_authorized_domain)
const;
663 const GoogleUrl&
base_url()
const {
return base_url_; }
672 StringPiece decoded_base()
const {
return decoded_base_url_.Spec(); }
675 bool IsHttps()
const {
return google_url().SchemeIs(
"https"); }
677 const UrlSegmentEncoder* default_encoder()
const {
return &default_encoder_; }
680 RewriteFilter*
FindFilter(
const StringPiece&
id)
const;
696 void set_containing_charset(
const StringPiece charset) {
697 charset.CopyToString(&containing_charset_);
704 HtmlResourceSlotPtr
GetSlot(
const ResourcePtr& resource,
706 HtmlElement::Attribute* attr);
712 InlineResourceSlotPtr
GetInlineSlot(
const ResourcePtr& resource,
713 HtmlCharactersNode* char_node);
720 const ResourcePtr& resource, HtmlElement* element,
721 HtmlElement::Attribute* attribute);
728 LOCKS_EXCLUDED(rewrite_mutex());
729 void InitiateFetch(RewriteContext* rewrite_context);
741 void RewriteComplete(RewriteContext* rewrite_context,
bool permit_render);
761 GoogleString
ToString(
bool show_detached_contexts)
const
762 LOCKS_EXCLUDED(rewrite_mutex());
763 GoogleString ToStringLockHeld(
bool show_detached_contexts)
const
764 EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
784 LOCKS_EXCLUDED(rewrite_mutex());
794 fully_rewrite_on_flush_ = x;
799 return fully_rewrite_on_flush_;
806 fast_blocking_rewrite_ = x;
809 bool fast_blocking_rewrite()
const {
810 return fast_blocking_rewrite_;
846 int rewrite_deadline_ms() {
return options()->rewrite_deadline_ms(); }
853 max_page_processing_delay_ms_ = x;
855 int max_page_processing_delay_ms() {
return max_page_processing_delay_ms_; }
859 UserAgentMatcher::DeviceType device_type()
const {
return device_type_; }
867 RewriteContext* candidate);
874 const GoogleString& partition_key, RewriteContext* candidate);
879 bool flush_requested()
const {
return flush_requested_; }
907 virtual void Flush();
921 QueuedWorkerPool::Sequence* html_worker() {
return html_worker_; }
922 Sequence* rewrite_worker();
923 Scheduler::Sequence* scheduler_sequence() {
924 return scheduler_sequence_.get();
927 QueuedWorkerPool::Sequence* low_priority_rewrite_worker() {
928 return low_priority_rewrite_worker_;
943 Scheduler* scheduler() {
return scheduler_; }
958 const StringPiece& output_css_base,
959 const StringPiece& contents,
961 MessageHandler* handler);
971 const GoogleUrl& output_base,
972 bool* proxy_mode)
const;
982 AbstractPropertyPage* page,
983 StringPiece property_name,
984 StringPiece property_value);
995 return fallback_property_page_;
1019 return critical_images_info_.get();
1027 return critical_selector_info_.get();
1034 critical_selector_info_.reset(info);
1042 critical_images_info_.reset(critical_images_info);
1048 return (
options()->Enabled(RewriteOptions::kFlattenCssImports) ||
1049 (!
options()->Forbidden(RewriteOptions::kFlattenCssImports) &&
1050 (
options()->Enabled(RewriteOptions::kPrioritizeCriticalCss) ||
1051 options()->Enabled(RewriteOptions::kComputeCriticalCss))));
1065 return num_flushed_early_pagespeed_resources_;
1071 ++num_flushed_early_pagespeed_resources_;
1093 void set_flushed_early(
bool x) { flushed_early_ = x; }
1094 bool flushed_early()
const {
return flushed_early_; }
1096 void set_flushing_early(
bool x) { flushing_early_ = x; }
1097 bool flushing_early()
const {
return flushing_early_; }
1099 void set_is_lazyload_script_flushed(
bool x) {
1100 is_lazyload_script_flushed_ = x;
1102 bool is_lazyload_script_flushed()
const {
1103 return is_lazyload_script_flushed_; }
1108 FlushEarlyRenderInfo* flush_early_render_info()
const;
1113 return dependency_tracker_.get();
1119 FlushEarlyRenderInfo* flush_early_render_info);
1131 void InsertDebugComments(
1132 const protobuf::RepeatedPtrField<GoogleString>& unescaped_messages,
1133 HtmlElement* element);
1134 void InsertUnauthorizedDomainDebugComment(StringPiece url,
1135 HtmlElement* element);
1139 const GoogleUrl& gurl);
1150 return dom_stats_filter_;
1180 bool Write(
const ResourceVector& inputs,
1181 const StringPiece& contents,
1182 const ContentType* type,
1183 StringPiece charset,
1186 void set_defer_instrumentation_script(
bool x) {
1187 defer_instrumentation_script_ = x;
1189 bool defer_instrumentation_script()
const {
1190 return defer_instrumentation_script_;
1196 ScopedMutex lock(rewrite_mutex());
1197 num_initiated_rewrites_ = x;
1199 int64 num_initiated_rewrites()
const {
1200 ScopedMutex lock(rewrite_mutex());
1201 return num_initiated_rewrites_;
1205 ScopedMutex lock(rewrite_mutex());
1206 num_detached_rewrites_ = x;
1208 int64 num_detached_rewrites()
const {
1209 ScopedMutex lock(rewrite_mutex());
1210 return num_detached_rewrites_;
1213 void set_pagespeed_query_params(StringPiece x) {
1214 x.CopyToString(&pagespeed_query_params_);
1216 StringPiece pagespeed_query_params()
const {
1217 return pagespeed_query_params_;
1220 void set_pagespeed_option_cookies(StringPiece x) {
1221 x.CopyToString(&pagespeed_option_cookies_);
1223 StringPiece pagespeed_option_cookies()
const {
1224 return pagespeed_option_cookies_;
1242 bool Decode(StringPiece leaf, ResourceNamer* resource_namer)
const;
1244 bool filters_added()
const {
return filters_added_; }
1245 bool has_html_writer_filter()
const {
1246 return html_writer_filter_.get() !=
nullptr;
1253 bool is_amp_document()
const {
return is_amp_; }
1256 virtual void DetermineFiltersBehaviorImpl();
1259 friend class DistributedRewriteContextTest;
1260 friend class RewriteContext;
1261 friend class RewriteDriverTest;
1262 friend class RewriteTestBase;
1263 friend class ServerContextTest;
1265 typedef std::map<GoogleString, RewriteFilter*> StringFilterMap;
1268 bool ShouldDistributeFetch(
const StringPiece& filter_id);
1281 bool DistributeFetch(
const StringPiece& url,
const StringPiece& filter_id,
1282 AsyncFetch* async_fetch);
1288 void CheckForCompletionAsync(
WaitMode wait_mode, int64 timeout_ms,
1290 EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
1295 void TryCheckForCompletion(
WaitMode wait_mode, int64 end_time_ms,
1297 EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
1300 bool IsDone(
WaitMode wait_mode,
bool deadline_reached)
1301 EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
1305 bool WaitForPendingAsyncEvents(
WaitMode wait_mode) {
1307 (fully_rewrite_on_flush_ && !fast_blocking_rewrite_);
1313 void FlushAsyncDone(
int num_rewrites, Function* callback);
1319 int64 ComputeCurrentFlushWindowRewriteDelayMs();
1322 void QueueFlushAsyncDone(
int num_rewrites, Function* callback);
1326 void QueueFinishParseAfterFlush(Function* user_callback);
1327 void FinishParseAfterFlush(Function* user_callback);
1329 bool RewritesComplete() const EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
1333 void SetBaseUrlIfUnset(const StringPiece& new_base);
1337 void SetBaseUrlForFetch(const StringPiece& url);
1341 void SetDecodedUrlFromBase();
1344 AbstractMutex* rewrite_mutex() const LOCK_RETURNED(scheduler_->mutex()) {
1345 return scheduler_->mutex();
1349 virtual void ParseTextInternal(
const char* content,
int size);
1352 bool ShouldSkipParsing();
1355 int SignatureLength()
const;
1357 friend class ScanFilter;
1362 void RegisterRewriteFilter(RewriteFilter* filter);
1368 void EnableRewriteFilter(
const char*
id);
1375 ResourcePtr CreateInputResourceUnchecked(
const GoogleUrl& gurl,
1376 bool is_authorized_domain);
1378 void AddPreRenderFilters();
1379 void AddPostRenderFilters();
1382 bool DecodeOutputResourceNameHelper(
const GoogleUrl& url,
1383 const RewriteOptions* options_to_use,
1384 const UrlNamer* url_namer,
1385 ResourceNamer* name_out,
1387 RewriteFilter** filter_out,
1388 GoogleString* url_base,
1389 StringVector* urls)
const;
1400 void WriteDomCohortIntoPropertyCache();
1403 CacheUrlAsyncFetcher* CreateCustomCacheFetcher(UrlAsyncFetcher* base_fetcher);
1411 void PossiblyPurgeCachedResponseAndReleaseDriver();
1431 bool PrepareShouldSignal() EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
1432 void SignalIfRequired(
bool result_of_prepare_should_signal)
1433 EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
1437 void CleanupRequestThread();
1456 bool refs_before_base_;
1459 GoogleString containing_charset_;
1463 void PopulateRequestContext();
1465 bool filters_added_;
1466 bool externally_managed_;
1483 kRefPendingRewrites,
1488 kRefDetachedRewrites,
1497 kRefDeletingRewrites,
1500 kRefFetchUserFacing,
1503 kRefFetchBackground,
1513 kRefRenderBlockingAsyncEvents,
1518 friend class CategorizedRefcount<RewriteDriver, RefCategory>;
1521 CategorizedRefcount<RewriteDriver, RefCategory> ref_counts_;
1524 void LastRefRemoved();
1525 StringPiece RefCategoryName(RefCategory cat);
1529 void DropReference(RefCategory cat);
1533 bool release_driver_;
1538 WaitMode waiting_ GUARDED_BY(rewrite_mutex());
1541 bool waiting_deadline_reached_ GUARDED_BY(rewrite_mutex());
1547 bool fully_rewrite_on_flush_;
1551 bool fast_blocking_rewrite_;
1553 bool flush_requested_;
1554 bool flush_occurred_;
1558 bool flushed_early_;
1562 bool flushing_early_;
1566 bool is_lazyload_script_flushed_;
1571 bool write_property_cache_dom_cohort_;
1575 GoogleUrl base_url_;
1580 GoogleUrl decoded_base_url_;
1584 GoogleString fetch_url_;
1586 GoogleString user_agent_;
1588 LazyBool should_skip_parsing_;
1590 StringFilterMap resource_filter_map_;
1592 ResponseHeaders* response_headers_;
1596 scoped_ptr<const RequestHeaders> request_headers_;
1602 typedef std::vector<RewriteContext*> RewriteContextVector;
1603 RewriteContextVector rewrites_;
1607 int max_page_processing_delay_ms_;
1609 typedef std::set<RewriteContext*> RewriteContextSet;
1615 RewriteContextSet initiated_rewrites_ GUARDED_BY(rewrite_mutex());
1618 int64 num_initiated_rewrites_ GUARDED_BY(rewrite_mutex());
1627 int64 num_detached_rewrites_ GUARDED_BY(rewrite_mutex());
1636 RewriteContextSet detached_rewrites_ GUARDED_BY(rewrite_mutex());
1639 int possibly_quick_rewrites_ GUARDED_BY(rewrite_mutex());
1643 RewriteContextVector fetch_rewrites_;
1647 FileSystem* file_system_;
1648 ServerContext* server_context_;
1649 Scheduler* scheduler_;
1650 UrlAsyncFetcher* default_url_async_fetcher_;
1655 UrlAsyncFetcher* url_async_fetcher_;
1660 UrlAsyncFetcher* distributed_async_fetcher_;
1664 std::vector<UrlAsyncFetcher*> owned_url_async_fetchers_;
1666 DomStatsFilter* dom_stats_filter_;
1667 scoped_ptr<HtmlWriterFilter> html_writer_filter_;
1669 ScanFilter scan_filter_;
1670 scoped_ptr<DomainRewriteFilter> domain_rewriter_;
1671 scoped_ptr<UrlLeftTrimFilter> url_trim_filter_;
1675 typedef std::map<GoogleString, RewriteContext*> PrimaryRewriteContextMap;
1676 PrimaryRewriteContextMap primary_rewrite_context_map_;
1678 HtmlResourceSlotSet slots_;
1679 InlineResourceSlotSet inline_slots_;
1680 InlineAttributeSlotSet inline_attribute_slots_;
1682 scoped_ptr<RewriteOptions> options_;
1684 RewriteDriverPool* controlling_pool_;
1687 scoped_ptr<CacheUrlAsyncFetcher::AsyncOpHooks>
1688 cache_url_async_fetcher_async_op_hooks_;
1691 UrlSegmentEncoder default_encoder_;
1694 FilterList early_pre_render_filters_;
1696 FilterList pre_render_filters_;
1699 std::vector<ResourceUrlClaimant*> resource_claimants_;
1704 FilterVector filters_to_delete_;
1706 QueuedWorkerPool::Sequence* html_worker_;
1707 QueuedWorkerPool::Sequence* rewrite_worker_;
1708 QueuedWorkerPool::Sequence* low_priority_rewrite_worker_;
1709 scoped_ptr<Scheduler::Sequence> scheduler_sequence_;
1715 FallbackPropertyPage* fallback_property_page_;
1718 bool owns_property_page_;
1721 scoped_ptr<PropertyPage> origin_property_page_;
1724 UserAgentMatcher::DeviceType device_type_;
1728 scoped_ptr<CriticalImagesInfo> critical_images_info_;
1729 scoped_ptr<CriticalSelectorInfo> critical_selector_info_;
1732 bool xhtml_mimetype_computed_;
1737 int num_inline_preview_images_;
1740 int num_flushed_early_pagespeed_resources_;
1745 DebugFilter* debug_filter_;
1747 scoped_ptr<FlushEarlyInfo> flush_early_info_;
1748 scoped_ptr<FlushEarlyRenderInfo> flush_early_render_info_;
1749 scoped_ptr<DependencyTracker> dependency_tracker_;
1751 bool can_rewrite_resources_;
1756 RequestContextPtr request_context_;
1759 int64 start_time_ms_;
1761 scoped_ptr<RequestProperties> request_properties_;
1766 static int initialized_count_;
1770 bool tried_to_distribute_fetch_;
1774 bool defer_instrumentation_script_;
1780 AtomicBool executing_rewrite_tasks_;
1783 DownstreamCachePurger downstream_cache_purger_;
1786 GoogleString pagespeed_query_params_;
1789 GoogleString pagespeed_option_cookies_;
1800 const ResponseHeaders& headers);
1809 const RequestContextPtr& request_ctx,
1810 const ResponseHeaders& headers);
1817 const RequestContextPtr& request_ctx);
virtual ResponseHeaders::VaryOption RespectVaryOnResources() const
OutputResourcePtr DecodeOutputResource(const GoogleUrl &url, RewriteFilter **filter) const
Definition: rewrite_driver.h:113
static const char kDomCohort[]
Definition: rewrite_driver.h:149
void EnableBlockingRewrite(RequestHeaders *request_headers)
void RunTasksOnRequestThread()
void set_max_page_processing_delay_ms(int x)
Definition: rewrite_driver.h:852
void set_flush_early_render_info(FlushEarlyRenderInfo *flush_early_render_info)
InlineResourceSlotPtr GetInlineSlot(const ResourcePtr &resource, HtmlCharactersNode *char_node)
void PopulateResourceNamer(const StringPiece &filter_id, const StringPiece &name, ResourceNamer *full_name)
const ResponseHeaders * response_headers()
Definition: rewrite_driver.h:254
OutputResourcePtr CreateOutputResourceWithPath(const StringPiece &mapped_path, const StringPiece &unmapped_path, const StringPiece &base_url, const StringPiece &filter_id, const StringPiece &name, OutputResourceKind kind, GoogleString *failure_reason)
void RewriteComplete(RewriteContext *rewrite_context, bool permit_render)
DomainRewriteFilter * domain_rewriter()
Definition: rewrite_driver.h:947
void ExecuteFlushIfRequested()
Definition: critical_selector_finder.h:43
bool Write(const ResourceVector &inputs, const StringPiece &contents, const ContentType *type, StringPiece charset, OutputResource *output)
bool DecodeUrlGivenOptions(const GoogleUrl &url, const RewriteOptions *options, const UrlNamer *url_namer, StringVector *decoded_urls) const
bool is_critical_images_beacon_enabled()
void AppendRewriteFilter(RewriteFilter *filter)
bool MayCacheExtendCss() const
bool MetadataRequested(const RequestHeaders &request_headers) const
void ClearRequestProperties()
Reinitializes request_properties_, clearing any cached values.
const GoogleUrl & base_url() const
Definition: rewrite_driver.h:663
void SwitchToQueuedWorkerPool() EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex())
bool FetchOutputResource(const OutputResourcePtr &output_resource, RewriteFilter *filter, AsyncFetch *async_fetch)
virtual bool IsCacheValid(const GoogleString &key, const ResponseHeaders &headers)
void AddOwnedPostRenderFilter(HtmlFilter *filter)
Adds a filter to the end of the post-render chain, taking ownership.
void set_options_for_pool(RewriteDriverPool *pool, RewriteOptions *options)
Definition: rewrite_driver.h:481
RewriteFilter * FindFilter(const StringPiece &id) const
Finds a filter with the given ID, or returns NULL if none found.
static const char kSubresourcesPropertyName[]
Flush Subresources Info associted with the HTML page.
Definition: rewrite_driver.h:163
void set_response_headers_ptr(ResponseHeaders *headers)
Definition: rewrite_driver.h:261
InlineAttributeSlotPtr GetInlineAttributeSlot(const ResourcePtr &resource, HtmlElement *element, HtmlElement::Attribute *attribute)
void SaveOriginalHeaders(const ResponseHeaders &response_headers)
void InfoAt(const RewriteContext *context, const char *msg,...) INSTAWEB_PRINTF_FORMAT(3
DependencyTracker * dependency_tracker() const
Definition: rewrite_driver.h:1112
bool tried_to_distribute_fetch() const
Did the driver attempt to distribute the fetch?
Definition: rewrite_driver.h:1166
CssResolutionStatus
Status return-code for ResolveCssUrls.
Definition: rewrite_driver.h:101
CriticalSelectorInfo * critical_selector_info()
Definition: rewrite_driver.h:1026
FallbackPropertyPage * fallback_property_page() const
Definition: rewrite_driver.h:994
StringPiece containing_charset()
Definition: rewrite_driver.h:695
PropertyPage * origin_property_page() const
ResourcePtr CreateInputResourceAbsoluteUncheckedForTestsOnly(const StringPiece &absolute_url)
FlushEarlyInfo * flush_early_info()
This method is not thread-safe. Call it only from the html parser thread.
void AddRewriteTask(Function *task)
Queues up a task to run on the (high-priority) rewrite thread.
static void InitStats(Statistics *statistics)
Initialize statistics for all filters that need it.
void SetSessionFetcher(UrlAsyncFetcher *f)
void IncrementAsyncEventsCount()
bool DebugMode() const
Definition: rewrite_driver.h:1123
Definition: url_left_trim_filter.h:47
bool FetchResource(const StringPiece &url, AsyncFetch *fetch)
void set_fully_rewrite_on_flush(bool x)
Definition: rewrite_driver.h:793
bool is_nested() const
Determine whether this driver is nested inside another.
Definition: rewrite_driver.h:1158
void increment_num_inline_preview_images()
We expect to this method to be called on the HTML parser thread.
bool LookupMetadataForOutputResource(StringPiece url, GoogleString *error_out, RewriteContext::CacheLookupResultCallback *callback)
void set_critical_images_info(CriticalImagesInfo *critical_images_info)
Definition: rewrite_driver.h:1041
void set_origin_property_page(PropertyPage *page)
Takes ownership of page.
void FlushAsync(Function *done)
void FinishParseAsync(Function *callback)
void DecrementAsyncEventsCount()
Decrements a reference count bumped up by IncrementAsyncEventsCount()
void AddOwnedEarlyPreRenderFilter(HtmlFilter *filter)
static void Initialize()
Initialize statics. Initialize/Terminate calls must be paired.
bool IsResourceUrlClaimed(const GoogleUrl &url) const
RequestTrace * trace_context()
void set_custom_options(RewriteOptions *options)
Takes ownership of 'options'.
Definition: rewrite_driver.h:475
void PrintStateToErrorLog(bool show_detached_contexts)
For logs.
XhtmlStatus MimeTypeXhtmlStatus()
void DetachedFetchComplete()
WaitMode
Mode for BoundedWaitForCompletion.
Definition: rewrite_driver.h:108
void SetIsAmpDocument(bool is_amp)
bool MatchesBaseUrl(const GoogleUrl &input_url) const
bool FlattenCssImportsEnabled() const
Definition: rewrite_driver.h:1047
Callback2< const GoogleUrl &, bool * > ResourceUrlClaimant
Definition: resource.h:357
Definition: rewrite_driver_pool.h:34
void SetRequestHeaders(const RequestHeaders &headers)
static GoogleString DeadlineExceededMessage(StringPiece filter_name)
Formats a "deadline exceeded" message for a given filter.
bool DecodeOutputResourceName(const GoogleUrl &url, const RewriteOptions *options_to_use, const UrlNamer *url_namer, ResourceNamer *name_out, OutputResourceKind *kind_out, RewriteFilter **filter_out) const
void FetchInPlaceResource(const GoogleUrl &gurl, bool proxy_mode, AsyncFetch *async_fetch)
static const char kStatusCodePropertyName[]
Status codes of previous responses.
Definition: rewrite_driver.h:165
void set_num_detached_rewrites(int64 x)
Sets the num_detached_rewrites_. This should only be called from test code.
Definition: rewrite_driver.h:1204
void DeleteRewriteContext(RewriteContext *rewrite_context)
OutputResourcePtr CreateOutputResourceWithMappedPath(const StringPiece &mapped_path, const StringPiece &unmapped_path, const StringPiece &filter_id, const StringPiece &name, OutputResourceKind kind, GoogleString *failure_reason)
Definition: rewrite_driver.h:584
void set_fast_blocking_rewrite(bool x)
Definition: rewrite_driver.h:805
void set_unowned_fallback_property_page(FallbackPropertyPage *page)
Does not take the ownership of the page.
ResponseHeaders * mutable_response_headers()
Definition: rewrite_driver.h:244
CssResolutionStatus ResolveCssUrls(const GoogleUrl &input_css_base, const StringPiece &output_css_base, const StringPiece &contents, Writer *writer, MessageHandler *handler)
XhtmlStatus
Definition: rewrite_driver.h:128
void set_fallback_property_page(FallbackPropertyPage *page)
Takes ownership of page.
void set_critical_selector_info(CriticalSelectorInfo *info)
Definition: rewrite_driver.h:1033
Definition: rewrite_driver.h:111
bool fully_rewrite_on_flush() const
Returns if this response has a blocking rewrite or not.
Definition: rewrite_driver.h:798
CacheUrlAsyncFetcher * CreateCacheOnlyFetcher()
Returns a cache fetcher that does not fall back to an actual fetcher.
ResourcePtr CreateInputResource(const GoogleUrl &input_url, bool *is_authorized)
void DeregisterForPartitionKey(const GoogleString &partition_key, RewriteContext *candidate)
static const char kLastRequestTimestamp[]
Definition: rewrite_driver.h:159
void AppendOwnedPreRenderFilter(HtmlFilter *filter)
Adds a filter to the end of the pre-render chain, taking ownership.
void DecrementRenderBlockingAsyncEventsCount()
bool refs_before_base()
Returns refs_before_base.
Definition: rewrite_driver.h:683
PropertyPage * property_page() const
int num_flushed_early_pagespeed_resources() const
Definition: rewrite_driver.h:1064
bool MayRewriteUrl(const GoogleUrl &domain_url, const GoogleUrl &input_url, InlineAuthorizationPolicy inline_authorization_policy, IntendedFor intended_for, bool *is_authorized_domain) const
bool can_rewrite_resources() const
Definition: rewrite_driver.h:1155
Definition: cache_url_async_fetcher.h:61
static GoogleString GenerateUnauthorizedDomainDebugComment(const GoogleUrl &gurl)
Generates an unauthorized domain debug comment. Public for unit tests.
StringPiece fetch_url() const
The URL that was requested if FetchResource was called.
Definition: rewrite_driver.h:666
OutputResourcePtr CreateOutputResourceFromResource(const char *filter_id, const UrlSegmentEncoder *encoder, const ResourceContext *data, const ResourcePtr &input_resource, OutputResourceKind kind, GoogleString *failure_reason)
void BoundedWaitFor(WaitMode mode, int64 timeout_ms) LOCKS_EXCLUDED(rewrite_mutex())
void AddUnownedPostRenderFilter(HtmlFilter *filter)
Same, without taking ownership.
bool IsHttps() const
Quick way to tell if the document url is https (ie was fetched via https).
Definition: rewrite_driver.h:675
GoogleString ToString(bool show_detached_contexts) const LOCKS_EXCLUDED(rewrite_mutex())
Debugging routines to print out data about the driver.
void ReportSlowRewrites(int num)
void PrependRewriteFilter(RewriteFilter *filter)
HtmlResourceSlotPtr GetSlot(const ResourcePtr &resource, HtmlElement *elt, HtmlElement::Attribute *attr)
Definition: rewrite_driver.h:98
const GoogleUrl & decoded_base_url() const
Definition: rewrite_driver.h:671
bool SetOrClearPageSpeedOptionCookies(const GoogleUrl &gurl, ResponseHeaders *response_headers)
void set_num_initiated_rewrites(int64 x)
Definition: rewrite_driver.h:1195
void set_property_page(PropertyPage *page)
Takes ownership of page.
void PrependOwnedPreRenderFilter(HtmlFilter *filter)
Adds a filter to the beginning of the pre-render chain, taking ownership.
Definition: server_context.h:100
virtual bool StartParseId(const StringPiece &url, const StringPiece &id, const ContentType &content_type)
RewriteDriverPool * controlling_pool()
Pool in which this driver can be recycled. May be NULL.
Definition: rewrite_driver.h:487
Definition: rewrite_context.h:146
void set_externally_managed(bool x)
Definition: rewrite_driver.h:823
OutputResourcePtr CreateOutputResourceWithUnmappedUrl(const GoogleUrl &unmapped_gurl, const StringPiece &filter_id, const StringPiece &name, OutputResourceKind kind, GoogleString *failure_reason)
virtual void FinishParse()
void AppendUnownedPreRenderFilter(HtmlFilter *filter)
Same, without taking ownership.
bool InitiateRewrite(RewriteContext *rewrite_context) LOCKS_EXCLUDED(rewrite_mutex())
void PrintState(bool show_detached_contexts)
For debugging.
int num_inline_preview_images() const
Definition: rewrite_driver.h:1057
void SetWriter(Writer *writer)
bool ShouldAbsolutifyUrl(const GoogleUrl &input_base, const GoogleUrl &output_base, bool *proxy_mode) const
void AddResourceUrlClaimant(ResourceUrlClaimant *claimant)
void bool GenerateOutputResourceNameAndUrl(const UrlSegmentEncoder *encoder, const ResourceContext *data, const ResourcePtr &input_resource, GoogleString *name, GoogleUrl *mapped_gurl, GoogleString *failure_reason)
Constructs name and URL for the specified input resource and encoder.
void AddLowPriorityRewriteTask(Function *task)
OptionsAwareHTTPCacheCallback(const RewriteOptions *rewrite_options, const RequestContextPtr &request_ctx)
void TracePrintf(const char *fmt,...)
InlineAuthorizationPolicy
See CreateInputResource.
Definition: rewrite_driver.h:135
const GoogleString & CacheFragment() const
wait for everything to complete (up to deadline)
Definition: rewrite_driver.h:110
static const char kBeaconCohort[]
The cohort for properties that are written by the beacon handler.
Definition: rewrite_driver.h:151
bool Decode(StringPiece leaf, ResourceNamer *resource_namer) const
Definition: http_cache.h:113
virtual int64 OverrideCacheTtlMs(const GoogleString &key)
OutputResourcePtr CreateOutputResourceWithPath(const StringPiece &path, const StringPiece &filter_id, const StringPiece &name, OutputResourceKind kind, GoogleString *failure_reason)
Definition: rewrite_driver.h:595
Definition: output_resource.h:44
CacheUrlAsyncFetcher * CreateCacheFetcher()
void set_refs_before_base()
Definition: rewrite_driver.h:689
RewriteContext * RegisterForPartitionKey(const GoogleString &partition_key, RewriteContext *candidate)
Used internally. Do not pass in.
Definition: rewrite_driver.h:109
Definition: dependency_tracker.h:41
void ExecuteFlushIfRequestedAsync(Function *callback)
AbstractLogRecord * log_record()
Definition: domain_rewrite_filter.h:50
void SetServerContext(ServerContext *server_context)
static const char kDependenciesCohort[]
Definition: rewrite_driver.h:155
const RewriteOptions * options() const
Return the options used for this RewriteDriver.
Definition: rewrite_driver.h:490
void UpdatePropertyValueInDomCohort(AbstractPropertyPage *page, StringPiece property_name, StringPiece property_value)
Definition: rewrite_options.h:84
void set_device_type(UserAgentMatcher::DeviceType x)
Sets the device type chosen for the current property_page.
Definition: rewrite_driver.h:858
Definition: rewrite_driver.h:1796
Counts some basic statistics observed as HTML is parsed.
Definition: dom_stats_filter.h:34
static const char kParseSizeLimitExceeded[]
Tracks if we exceeded the maximum size limit of html which we should parse.
Definition: rewrite_driver.h:161
bool DecodeUrl(const GoogleUrl &url, StringVector *decoded_urls) const
Decodes the incoming pagespeed url to original url(s).
Definition: request_properties.h:37
void InsertDebugComment(StringPiece unescaped_message, HtmlNode *node)
CriticalImagesInfo * critical_images_info() const
Used by ImageRewriteFilter for identifying critical images.
Definition: rewrite_driver.h:1018
void increment_num_flushed_early_pagespeed_resources()
Definition: rewrite_driver.h:1070
IntendedFor
See CreateInputResource.
Definition: rewrite_driver.h:141
Definition: url_async_fetcher.h:33
void IncrementRenderBlockingAsyncEventsCount()
Definition: resource_namer.h:32
void set_distributed_fetcher(UrlAsyncFetcher *fetcher)
Does not take ownership.
Definition: rewrite_driver.h:461
Definition: critical_images_finder.h:52
void RequestFlush()
Definition: rewrite_driver.h:878
OutputResourceKind
Definition: output_resource_kind.h:26