Page Speed Optimization Libraries
1.11.33.2
|
Unit-test framework for wget fetcher. More...
Namespaces | |
css_util | |
Typedefs | |
typedef std::map< GoogleString, std::pair< int32, int32 > > | RenderedImageDimensionsMap |
typedef RefCountedPtr < CssResourceSlot > | CssResourceSlotPtr |
typedef RefCountedPtr < InlineAttributeSlot > | InlineAttributeSlotPtr |
typedef std::set < InlineAttributeSlotPtr, InlineAttributeSlotComparator > | InlineAttributeSlotSet |
typedef RefCountedPtr < InlineResourceSlot > | InlineResourceSlotPtr |
typedef std::set < InlineResourceSlotPtr, InlineResourceSlotComparator > | InlineResourceSlotSet |
typedef RefCountedPtr< Resource > | ResourcePtr |
typedef std::vector< ResourcePtr > | ResourceVector |
typedef Callback2< const GoogleUrl &, bool * > | ResourceUrlClaimant |
typedef RefCountedPtr < ResourceSlot > | ResourceSlotPtr |
typedef RefCountedPtr < HtmlResourceSlot > | HtmlResourceSlotPtr |
typedef std::vector < ResourceSlotPtr > | ResourceSlotVector |
typedef std::set < HtmlResourceSlotPtr, HtmlResourceSlotComparator > | HtmlResourceSlotSet |
typedef std::vector < ResponsiveImageCandidate > | ResponsiveImageCandidateVector |
typedef std::map< HtmlElement *, ResponsiveVirtualImages > | ResponsiveImageCandidateMap |
typedef RefCountedPtr < OutputResource > | OutputResourcePtr |
typedef std::vector < OutputResourcePtr > | OutputResourceVector |
typedef bool(* | WebpProgressHook )(int percent, void *user_data) |
Progress hook for WebP conversions. | |
Enumerations | |
enum | FetchResponseStatus { kFetchStatusNotSet = 0, kFetchStatusOK = 1, kFetchStatusUncacheable200 = 2, kFetchStatusUncacheableError = 3, kFetchStatus4xxError = 4, kFetchStatusOtherError = 5, kFetchStatusDropped = 6, kFetchStatusEmpty = 7 } |
enum | BeaconStatus { kDoNotBeacon, kBeaconNoNonce, kBeaconWithNonce } |
enum | CriticalKeysWriteFlags { kNoRequirementsOnPriorResult = 0, kReplacePriorResult = 1, kRequirePriorSupport = 2, kSkipNonceCheck = 4 } |
enum | InlineResult { INLINE_SUCCESS, INLINE_UNSUPPORTED_DEVICE, INLINE_NOT_CRITICAL, INLINE_NO_DATA, INLINE_TOO_LARGE, INLINE_CACHE_SMALL_IMAGES_UNREWRITTEN, INLINE_RESPONSIVE, INLINE_INTERNAL_ERROR } |
enum | OutputResourceKind { kRewrittenResource, kOnTheFlyResource, kInlineResource, kOutlinedResource } |
enum | PropertyCacheDecodeResult { kPropertyCacheDecodeNotFound, kPropertyCacheDecodeExpired, kPropertyCacheDecodeParseError, kPropertyCacheDecodeOk } |
enum | PropertyCacheUpdateResult { kPropertyCacheUpdateNotFound, kPropertyCacheUpdateEncodeError, kPropertyCacheUpdateOk } |
enum | RewriteResult { kRewriteFailed, kRewriteOk, kTooBusy } |
Functions | |
bool | ValidateAndExpireNonce (int64 now_ms, StringPiece nonce, CriticalKeys *critical_keys) |
void | GetCriticalKeysFromProto (int64 support_percentage, const CriticalKeys &critical_keys, StringSet *keys) |
void | UpdateCriticalKeys (bool require_prior_support, const StringSet &new_set, int support_value, CriticalKeys *critical_keys) |
bool | ShouldBeacon (int64 next_beacon_timestamp_ms, const RewriteDriver &driver) |
void | WriteCriticalKeysToPropertyCache (const StringSet &new_keys, StringPiece nonce, int support_interval, CriticalKeysWriteFlags flags, StringPiece property_name, const PropertyCache *cache, const PropertyCache::Cohort *cohort, AbstractPropertyPage *page, MessageHandler *message_handler, Timer *timer) |
void | PrepareForBeaconInsertionHelper (CriticalKeys *proto, NonceGenerator *nonce_generator, RewriteDriver *driver, bool using_candidate_key_detection, BeaconMetadata *result) |
bool | UpdateCandidateKeys (const StringSet &keys, CriticalKeys *proto, bool clear_rebeacon_timestamp) |
bool | IsBeaconDataAvailable (const CriticalKeys &proto) |
Image * | NewImage (const StringPiece &original_contents, const GoogleString &url, const StringPiece &file_prefix, Image::CompressionOptions *options, Timer *timer, MessageHandler *handler) |
Image * | BlankImageWithOptions (int width, int height, ImageType type, const StringPiece &tmp_dir, Timer *timer, MessageHandler *handler, Image::CompressionOptions *options) |
int | CharToInt (char c) |
char to int without sign extension. More... | |
int | JpegIntAtPosition (const StringPiece &buf, size_t pos) |
int | GifIntAtPosition (const StringPiece &buf, size_t pos) |
int | PngIntAtPosition (const StringPiece &buf, size_t pos) |
bool | PngSectionIdIs (const char *hdr, const StringPiece &buf, size_t pos) |
const PropertyValue * | DecodeFromPropertyCacheHelper (const PropertyCache *cache, AbstractPropertyPage *page, const PropertyCache::Cohort *cohort, StringPiece property_name, int64 cache_ttl_ms, PropertyCacheDecodeResult *status) |
template<typename T > | |
T * | DecodeFromPropertyCache (const PropertyCache *cache, AbstractPropertyPage *page, const PropertyCache::Cohort *cohort, StringPiece property_name, int64 cache_ttl_ms, PropertyCacheDecodeResult *status) |
template<typename T > | |
T * | DecodeFromPropertyCache (RewriteDriver *driver, const PropertyCache::Cohort *cohort, StringPiece property_name, int64 cache_ttl_ms, PropertyCacheDecodeResult *status) |
PropertyCacheUpdateResult | UpdateInPropertyCache (const protobuf::MessageLite &value, const PropertyCache::Cohort *cohort, StringPiece property_name, bool write_cohort, AbstractPropertyPage *page) |
PropertyCacheUpdateResult | UpdateInPropertyCache (const protobuf::MessageLite &value, RewriteDriver *driver, const PropertyCache::Cohort *cohort, StringPiece property_name, bool write_cohort) |
bool | OptimizeWebp (const GoogleString &original_jpeg, int configured_quality, WebpProgressHook progress_hook, void *progress_hook_data, GoogleString *compressed_webp, MessageHandler *message_handler) |
bool | ReduceWebpImageQuality (const GoogleString &original_webp, int quality, GoogleString *compressed_webp) |
Variables | |
const char | kGaSnippetGetTracker [] = "_modpagespeed_getRewriteTracker(" |
const char | kGaSnippetPrefix [] |
const char | kGaSnippetSuffix [] |
const int64 | kBeaconTimeoutIntervalMs = Timer::kMinuteMs |
const int64 | kHighFreqBeaconCount = 3 |
const int64 | kLowFreqBeaconMult = 100 |
const int64 | kNonceExpirationLimit = 5 |
const char | kGAExperimentSnippet [] |
Visible only for use in tests. | |
const char | kGAJsSnippet [] |
const char | kAnalyticsJsSnippet [] |
const char | kAnalyticsJsIncreaseSiteSpeedTracking [] |
const char | kAnalyticsJsIncreaseSiteSpeedTrackingMinimal [] |
const char | kContentExperimentsJsClientUrl [] |
const char | kContentExperimentsNonNumericVariantComment [] |
const char | kContentExperimentsSetChosenVariationSnippet [] |
const char | kContentExperimentsSetExpAndVariantSnippet [] |
const char | kGASpeedTracking [] |
const int | kNoQualityGiven = -1 |
Unit-test framework for wget fetcher.
extern "C"
for Writer
Base class for tests which want a ServerContext.
for NULL
for StringSet, etc
Some common routines and constants for tests dealing with Images.
for GoogleString
A filter which does not modify the DOM, but counts statistics about it.
Base class for tests which do rewrites within CSS.
namespace Css
for ResourcePtr
DO NOT EDIT. Generated by ./google_analytics_snippet_gen.py.
Callbacks used for testing.
for size_t
for FILE
html_filter that passes data through unmodified, but logs statistics about the data as it goes by. It should be possible to create many instances of this class and insert them at different points in the rewriting flow Goal is to log: NUM_EXPLICIT_CLOSED - <tag> </tag> pairs NUM_IMPLICIT_CLOSED - <tag> for implicitly-closed tag NUM_BRIEF_CLOSED - </tag> NUM_CLOSED - Sum of above three NUM_UNCLOSED - <tag> without matching </tag> NUM_SPURIOUS_CLOSED - </tag> without preceding <tag>; UNCOUNTED RIGHT NOW! NUM_TAGS - Total number of opening tags NUM_CDATA - cdata sections NUM_COMMENTS - comments NUM_DIRECTIVES - directives NUM_DOCUMENTS - started documents NUM_IE_DIRECTIVES - ie directives Reporting: We report this information via a StatisticsLog: filter.ToString(log) Two sets of statistics (eg before and after processing) can be compared using before.Equals(after),
Copyright 2011 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
http:///www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
Helper class to make RewriteTestBase tests that use a custom options subclass.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
Search for synchronous loads of Google Analytics similar to the following:
<script type="text/javascript"> var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www."); document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js type='text/javascript'" + "%3E%3C/script%3E")); </script> <script type="text/javascript"> try { var pageTracker = _gat._getTracker("UA-XXXXX-X"); pageTracker._trackPageview(); } catch(err) {} </script>
Replace the document.write with a new snippet that loads ga.js asynchronously. Also, insert a replacement for _getTracker that converts any calls to the synchronous API to the asynchronous API. The _getTracker replacement is a new function that returns a mock tracker object. Anytime a synchronous API method is called, the mock tracker fowards it to a _gaq.push(...) call.
An alternative approach would been to find all the API calls and rewrite them to the asynchronous API. However, to be done properly, it would have had the added complication of using a JavaScript compiler.
Unit-test the RewriteContext class. This is made simplest by setting up some dummy rewriters in our test framework.
We need to include rewrite_driver.h due to covariant return of html_parse() We need to include mock_timer.h to allow upcast to Timer*.
typedef Callback2<const GoogleUrl&, bool*> net_instaweb::ResourceUrlClaimant |
Sometimes some portions of URL space need to be handled differently by dedicated resource subclasses. ResourceProvider callbacks are used to teach RewriteDriver about these, so it knows not to build regular UrlInputResource objects.
This enumerates different states of the fetched response. This module is mostly concerned with specific failure statuses, but it's convenient to have non-failure ones in the same enum.
Enumerator | |
---|---|
INLINE_RESPONSIVE |
Image should not be inlined because it is part of a responsive image. |
Used to signal whether optimization was successful or not to RewriteContext::RewriteDone.
Image* net_instaweb::BlankImageWithOptions | ( | int | width, |
int | height, | ||
ImageType | type, | ||
const StringPiece & | tmp_dir, | ||
Timer * | timer, | ||
MessageHandler * | handler, | ||
Image::CompressionOptions * | options | ||
) |
Creates a blank image of the given dimensions and type. For now, this is assumed to be an 8-bit 4-channel image transparent image.
|
inline |
char to int without sign extension.
The following four helper functions were moved here for testability. We ran into problems with sign extension under different compiler versions, and we'd like to catch regressions on that front in the future.
T* net_instaweb::DecodeFromPropertyCache | ( | const PropertyCache * | cache, |
AbstractPropertyPage * | page, | ||
const PropertyCache::Cohort * | cohort, | ||
StringPiece | property_name, | ||
int64 | cache_ttl_ms, | ||
PropertyCacheDecodeResult * | status | ||
) |
Decodes a protobuf of type T from the property named 'property_name' in the cohort 'cohort_name' in the given property cache, and makes sure it has not exceeded its TTL of 'cache_ttl_ms' (a value of -1 will disable this check).
*status will denote the decoding state; if it's kPropertyCacheDecodeOk then a pointer to a freshly allocated decoded proto is returned; otherwise NULL is returned.
*status set by helper
T* net_instaweb::DecodeFromPropertyCache | ( | RewriteDriver * | driver, |
const PropertyCache::Cohort * | cohort, | ||
StringPiece | property_name, | ||
int64 | cache_ttl_ms, | ||
PropertyCacheDecodeResult * | status | ||
) |
Wrapper version of the above function that gets the property cache and the property page from the given driver.
const PropertyValue* net_instaweb::DecodeFromPropertyCacheHelper | ( | const PropertyCache * | cache, |
AbstractPropertyPage * | page, | ||
const PropertyCache::Cohort * | cohort, | ||
StringPiece | property_name, | ||
int64 | cache_ttl_ms, | ||
PropertyCacheDecodeResult * | status | ||
) |
Returns PropertyValue object for given cohort and property name, setting *status and returning NULL if any errors were found.
void net_instaweb::GetCriticalKeysFromProto | ( | int64 | support_percentage, |
const CriticalKeys & | critical_keys, | ||
StringSet * | keys | ||
) |
Generate a list of the critical keys from a proto, storing it into keys. Takes into account legacy keys that may have been added before. A key is considered critical if its support is at least support_percentage of the maximum possible support value (which ramps up as beacon results arrive). When support_percentage = 0, any support is sufficient; when support_percentage = 100 all beacon results must support criticality.
|
inline |
Based on the CriticalKeys data seen so far, describe whether beacon metadata is available. This returns false until data is received.
Image* net_instaweb::NewImage | ( | const StringPiece & | original_contents, |
const GoogleString & | url, | ||
const StringPiece & | file_prefix, | ||
Image::CompressionOptions * | options, | ||
Timer * | timer, | ||
MessageHandler * | handler | ||
) |
Image owns none of its inputs. All of the arguments to NewImage(...) (the original_contents in particular) must outlive the Image object itself. The intent is that an Image is created in a scoped fashion from an existing known resource.
The options should be set via Image::SetOptions after construction, before the image is used for anything but determining its natural dimension size.
bool net_instaweb::OptimizeWebp | ( | const GoogleString & | original_jpeg, |
int | configured_quality, | ||
WebpProgressHook | progress_hook, | ||
void * | progress_hook_data, | ||
GoogleString * | compressed_webp, | ||
MessageHandler * | message_handler | ||
) |
Optimizer in the style of pagespeed/kernel/image/jpeg_optimizer.h that creates a webp-formatted image in compressed_webp from the jpeg image in original_jpeg. Indicates failure by returning false, in which case compressed_webp may be filled with junk.
void net_instaweb::PrepareForBeaconInsertionHelper | ( | CriticalKeys * | proto, |
NonceGenerator * | nonce_generator, | ||
RewriteDriver * | driver, | ||
bool | using_candidate_key_detection, | ||
BeaconMetadata * | result | ||
) |
Given a set of candidate critical keys, decide whether beaconing should take place. We should always beacon if there's new critical key data. Otherwise re-beaconing is based on a time and request interval, and 2 modes of beaconing frequency are supported. At first, beaconing occurs at a high frequency until we have collected kHighFreqBeaconCount beacons; after that, we transition into low frequency beaconing mode, where beaconing occurs less often. We also track the number of expired nonces since the last valid beacon was received to see if beaconing is set up correctly, and if it looks like it isn't, only do low frequency beaconing. Sets status and nonce appropriately in *result (nonce will be empty if no nonce is required). If candidate keys are not required, keys may be empty (but new candidate detection will not occur). If result->status != kDontBeacon, caller should write proto back to the property cache using UpdateInPropertyCache.
bool net_instaweb::ReduceWebpImageQuality | ( | const GoogleString & | original_webp, |
int | quality, | ||
GoogleString * | compressed_webp | ||
) |
Reduce the quality of the webp image. Indicates failure by returning false. WebP quality varies from 1 to 100. Original image will be returned if input quality is <1.
bool net_instaweb::UpdateCandidateKeys | ( | const StringSet & | keys, |
CriticalKeys * | proto, | ||
bool | clear_rebeacon_timestamp | ||
) |
Update the candidate key set in proto. If new candidate keys are detected, they are inserted into proto with a support value of 0, and true is returned. Otherwise returns false. If clear_rebeacon_timestamp is set, the rebeacon timestamp field in the proto is cleared to force rebeaconing on the next request.
void net_instaweb::UpdateCriticalKeys | ( | bool | require_prior_support, |
const StringSet & | new_set, | ||
int | support_value, | ||
CriticalKeys * | critical_keys | ||
) |
Add support for new_set to existing support. The new_set should be obtained from a fully-validated beacon result – this means PrepareForBeaconInsertion should have been called if required, and the resulting nonce should have been checked. If require_prior_support then there must be an existing support entry (possibly 0) for new support to be registered.
|
inline |
Updates the property 'property_name' in cohort 'cohort_name' of the property cache managed by the rewrite driver with the new value of the proto T. If 'write_cohort' is true, will also additionally write out the cohort to the cache backing.
bool net_instaweb::ValidateAndExpireNonce | ( | int64 | now_ms, |
StringPiece | nonce, | ||
CriticalKeys * | critical_keys | ||
) |
Check whether the given nonce is valid, invalidating any expired nonce entries we might encounter. To avoid the need to copy and clear the nonce list, we invalidate the entry used and any expired entries by clearing the nonce value and timestamp. These entries will be reused by AddNonceToCriticalSelectors.
void net_instaweb::WriteCriticalKeysToPropertyCache | ( | const StringSet & | new_keys, |
StringPiece | nonce, | ||
int | support_interval, | ||
CriticalKeysWriteFlags | flags, | ||
StringPiece | property_name, | ||
const PropertyCache * | cache, | ||
const PropertyCache::Cohort * | cohort, | ||
AbstractPropertyPage * | page, | ||
MessageHandler * | message_handler, | ||
Timer * | timer | ||
) |
Update the property cache with a new set of keys. This will update the support value for the new keys. If require_prior_support is set, any keys that are not already present in the property cache will be ignored (to prevent spurious keys from being injected). Note that it only increases the support value for the new keys, it does not decay values that are not present. PrepareForBeaconInsertion should have been called previously if !should_replace_prior_result and nonces must be checked.
const int64 net_instaweb::kBeaconTimeoutIntervalMs = Timer::kMinuteMs |
The amount of time after generating a nonce that we will accept it as valid. This keeps an attacker from accumulating large numbers of valid nonces to send many beacon responses at once.
const char net_instaweb::kGaSnippetSuffix[] |
const int64 net_instaweb::kHighFreqBeaconCount = 3 |
The number of valid beacons received that will switch from high frequency to low frequency beaconing.
const int64 net_instaweb::kLowFreqBeaconMult = 100 |
The multiplier to apply to RewriteOptions::beacon_reinstrument_time_sec() to determine the low frequency beaconing interval. For example, the default value rebeaconing value is 5 seconds, so we will rebeacon every 5 seconds in high frequency mode, and every 500 seconds (~8 minutes) in low frequency mode.
const int64 net_instaweb::kNonceExpirationLimit = 5 |
The limit on the number of nonces that can expire before we stop trying to do high frequency beaconing. This is a signal that beacons are not configured correctly and so we drop into low frequency beaconing mode.
const int net_instaweb::kNoQualityGiven = -1 |
Quality taken when no quality is passed through flags or when no quality is retrieved from JpegUtils::GetImageQualityFromImage.