corysanin/tubestation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge m-c to autoland. a=merge

Browse Source
This commit is contained in:
Ryan VanderMeulen
2017-08-09 18:51:26 -04:00
parent b305078f69 f72c834207
commit 29860c1c91
239 changed files with 28230 additions and 47931 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CLOBBER
View File

@@ -22,4 +22,4 @@
# changes to stick? As of bug 928195, this shouldn't be necessary! Please
# don't change CLOBBER for WebIDL changes any more.
Merge day clobber
Bug 1340910 - clobber required for Brotli 0.6.0 update due to changes in exported headers.

8
accessible/generic/DocAccessible.cpp
View File

@@ -2108,6 +2108,10 @@ DocAccessible::DoARIAOwnsRelocation(Accessible* aOwner)
"candidate", child, nullptr);
#endif
if (owned->IndexOf(child) < idx) {
continue; // ignore second entry of same ID
}
// Same child on same position, no change.
if (child->Parent() == aOwner) {
int32_t indexInParent = child->IndexInParent();
@@ -2138,10 +2142,6 @@ DocAccessible::DoARIAOwnsRelocation(Accessible* aOwner)
MOZ_ASSERT(owned->SafeElementAt(idx) != child, "Already in place!");
if (owned->IndexOf(child) < idx) {
continue; // ignore second entry of same ID
}
// A new child is found, check for loops.
if (child->Parent() != aOwner) {
// Child is aria-owned by another container, skip.

2
browser/base/content/browser-places.js
View File

@@ -462,7 +462,7 @@ var PlacesCommandHook = {
try {
title = docInfo.isErrorPage ? PlacesUtils.history.getPageTitle(uri)
: aBrowser.contentTitle;
title = title || uri.spec;
title = title || uri.displaySpec;
description = docInfo.description;
charset = aBrowser.characterSet;
} catch (e) { }

14
browser/base/content/browser.js
View File

@@ -934,7 +934,7 @@ function gKeywordURIFixup({ target: browser, data: fixupInfo }) {
browser = null;
// Additionally, we need the host of the parsed url
let hostName = alternativeURI.host;
let hostName = alternativeURI.displayHost;
// and the ascii-only host for the pref:
let asciiHost = alternativeURI.asciiHost;
// Normalize out a single trailing dot - NB: not using endsWith/lastIndexOf
@@ -2778,7 +2778,7 @@ function losslessDecodeURI(aURI) {
if (scheme == "moz-action")
throw new Error("losslessDecodeURI should never get a moz-action URI");
var value = aURI.spec;
var value = aURI.displaySpec;
let decodeASCIIOnly = !["https", "http", "file", "ftp"].includes(scheme);
// Try to decode as UTF-8 if there's no encoding sequence that we would break.
@@ -6726,7 +6726,7 @@ function warnAboutClosingWindow() {
var MailIntegration = {
sendLinkForBrowser(aBrowser) {
this.sendMessage(aBrowser.currentURI.spec, aBrowser.contentTitle);
this.sendMessage(aBrowser.currentURI.displaySpec, aBrowser.contentTitle);
},
sendMessage(aBody, aSubject) {
@@ -7805,7 +7805,7 @@ var gIdentityHandler = {
if (gURLBar.getAttribute("pageproxystate") != "valid")
return;
let value = gBrowser.currentURI.spec;
let value = gBrowser.currentURI.displaySpec;
let urlString = value + "\n" + gBrowser.contentTitle;
let htmlString = "<a href=\"" + value + "\">" + value + "</a>";
@@ -8166,13 +8166,13 @@ function switchToTabHavingURI(aURI, aOpenNew, aOpenParams = {}) {
// work correctly with URL objects - so treat them as strings
let ignoreFragmentWhenComparing = typeof ignoreFragment == "string" &&
ignoreFragment.startsWith("whenComparing");
let requestedCompare = cleanURL(
aURI.spec, ignoreQueryString || replaceQueryString, ignoreFragmentWhenComparing);
let requestedCompare = cleanURL(
aURI.displaySpec, ignoreQueryString || replaceQueryString, ignoreFragmentWhenComparing);
let browsers = aWindow.gBrowser.browsers;
for (let i = 0; i < browsers.length; i++) {
let browser = browsers[i];
let browserCompare = cleanURL(
browser.currentURI.spec, ignoreQueryString || replaceQueryString, ignoreFragmentWhenComparing);
browser.currentURI.displaySpec, ignoreQueryString || replaceQueryString, ignoreFragmentWhenComparing);
if (requestedCompare == browserCompare) {
aWindow.focus();
if (ignoreFragment == "whenComparingAndReplace" || replaceQueryString) {

10
browser/base/content/content.js
View File

@@ -1192,7 +1192,7 @@ var PageInfoListener = {
let hostName = null;
try {
hostName = window.location.host;
hostName = Services.io.newURI(window.location.href).displayHost;
} catch (exception) { }
windowInfo.hostName = hostName;
@@ -1203,7 +1203,15 @@ var PageInfoListener = {
let docInfo = {};
docInfo.title = document.title;
docInfo.location = document.location.toString();
try {
docInfo.location = Services.io.newURI(document.location.toString()).displaySpec;
} catch (exception) { }
docInfo.referrer = document.referrer;
try {
if (document.referrer) {
docInfo.referrer = Services.io.newURI(document.referrer).displaySpec;
}
} catch (exception) { }
docInfo.compatMode = document.compatMode;
docInfo.contentType = document.contentType;
docInfo.characterSet = document.characterSet;

2
browser/base/content/pageinfo/permissions.js
View File

@@ -41,7 +41,7 @@ function onLoadPermission(uri, principal) {
gPermURI = uri;
gPermPrincipal = principal;
var hostText = document.getElementById("hostText");
hostText.value = gPermURI.prePath;
hostText.value = gPermURI.displayPrePath;
for (var i of gPermissions)
initRow(i);

52
browser/base/content/tabbrowser.xml
View File

@@ -1514,11 +1514,11 @@
[ brandShortName ]);
isContentTitle = true;
} else {
if (browser.currentURI.spec) {
if (browser.currentURI.displaySpec) {
try {
title = this.mURIFixup.createExposableURI(browser.currentURI).spec;
title = this.mURIFixup.createExposableURI(browser.currentURI).displaySpec;
} catch (ex) {
title = browser.currentURI.spec;
title = browser.currentURI.displaySpec;
}
}
@@ -2219,14 +2219,10 @@
let setter;
switch (name) {
case "audioMuted":
getter = () => {
return false;
};
getter = () => false;
break;
case "contentTitle":
getter = () => {
return SessionStore.getLazyTabValue(aTab, "title");
};
getter = () => SessionStore.getLazyTabValue(aTab, "title");
break;
case "currentURI":
getter = () => {
@@ -2234,29 +2230,23 @@
return Services.io.newURI(url);
};
break;
case "fullZoom":
case "textZoom":
getter = () => 1;
break;
case "getTabBrowser":
getter = () => {
return () => {
return this;
};
};
getter = () => () => this;
break;
case "isRemoteBrowser":
getter = () => {
return browser.getAttribute("remote") == "true";
};
getter = () => browser.getAttribute("remote") == "true";
break;
case "permitUnload":
getter = () => {
return () => {
return { permitUnload: true, timedOut: false };
};
};
getter = () => () => ({ permitUnload: true, timedOut: false });
break;
case "reload":
case "reloadWithFlags":
getter = () => {
return (params) => {
getter = () =>
params => {
// Wait for load handler to be instantiated before
// initializing the reload.
aTab.addEventListener("SSTabRestoring", () => {
@@ -2264,25 +2254,21 @@
}, { once: true });
gBrowser._insertBrowser(aTab);
};
};
break;
case "resumeMedia":
getter = () => {
return () => {
getter = () =>
() => {
// No need to insert a browser, so we just call the browser's
// method.
aTab.addEventListener("SSTabRestoring", () => {
browser[name]();
}, { once: true });
};
};
break;
case "userTypedValue":
case "userTypedClear":
case "mediaBlocked":
getter = () => {
return SessionStore.getLazyTabValue(aTab, name);
};
getter = () => SessionStore.getLazyTabValue(aTab, name);
break;
default:
getter = () => {
@@ -2294,7 +2280,7 @@
this._insertBrowser(aTab);
return browser[name];
};
setter = (value) => {
setter = value => {
if (AppConstants.NIGHTLY_BUILD) {
let message =
`[bug 1345098] Lazy browser prematurely inserted via '${name}' property access:\n`
@@ -7933,7 +7919,7 @@
<![CDATA[
var tab = event.target.tab;
if (tab) {
let overLink = tab.linkedBrowser.currentURI.spec;
let overLink = tab.linkedBrowser.currentURI.displaySpec;
if (overLink == "about:blank")
overLink = "";
XULBrowserWindow.setOverLink(overLink, null);

2
browser/base/content/test/siteIdentity/browser_identity_UI.js
View File

@@ -33,7 +33,7 @@ var tests = [
},
{
name: "IDN subdomain",
location: "http://sub1." + idnDomain + "/",
location: "http://sub1.xn--hxajbheg2az3al.xn--jxalpdlp/",
effectiveHost: "sub1." + idnDomain
},
{

14
browser/base/content/test/urlbar/browser_urlbarCopying.js
View File

@@ -137,6 +137,19 @@ var tests = [
copyVal: "<example.com/\xe9>\xe9",
copyExpected: "http://example.com/\xe9"
},
{ // Note: it seems BrowserTestUtils.loadURI fails for unicode domains
loadURL: "http://sub2.xn--lt-uia.mochi.test:8888/foo",
expectedURL: toUnicode("sub2.ält.mochi.test:8888/foo"),
copyExpected: toUnicode("http://sub2.ält.mochi.test:8888/foo")
},
{
copyVal: toUnicode("s<ub2.ält.mochi.test:8888/f>oo"),
copyExpected: toUnicode("ub2.ält.mochi.test:8888/f")
},
{
copyVal: toUnicode("<sub2.ält.mochi.test:8888/f>oo"),
copyExpected: toUnicode("http://sub2.ält.mochi.test:8888/f")
},
{
loadURL: "http://example.com/?%C3%B7%C3%B7",
@@ -211,6 +224,7 @@ function runTest(testCase, cb) {
}
if (testCase.loadURL) {
info(`Loading : ${testCase.loadURL}\n`);
loadURL(testCase.loadURL, doCheck);
} else {
if (testCase.setURL)

10
browser/base/content/urlbarBindings.xml
View File

@@ -227,7 +227,7 @@ file, You can obtain one at http://mozilla.org/MPL/2.0/.
} else {
let originalUrl = ReaderMode.getOriginalUrlObjectForDisplay(aValue);
if (originalUrl) {
returnValue = originalUrl.spec;
returnValue = originalUrl.displaySpec;
}
}
@@ -981,7 +981,7 @@ file, You can obtain one at http://mozilla.org/MPL/2.0/.
// Avoid copying 'about:reader?url=', and always provide the original URI:
// Reader mode ensures we call createExposableURI itself.
let readerStrippedURI = ReaderMode.getOriginalUrlObjectForDisplay(uri.spec);
let readerStrippedURI = ReaderMode.getOriginalUrlObjectForDisplay(uri.displaySpec);
if (readerStrippedURI) {
uri = readerStrippedURI;
} else {
@@ -997,12 +997,12 @@ file, You can obtain one at http://mozilla.org/MPL/2.0/.
if (inputVal == selectedVal &&
!uri.schemeIs("javascript") && !uri.schemeIs("data") &&
!Services.prefs.getBoolPref("browser.urlbar.decodeURLsOnCopy")) {
return uri.spec;
return uri.displaySpec;
}
// Just the beginning of the URL is selected, or we want a decoded
// url. First check for a trimmed value.
let spec = uri.spec;
let spec = uri.displaySpec;
let trimmedSpec = this.trimValue(spec);
if (spec != trimmedSpec) {
// Prepend the portion that trimValue removed from the beginning.
@@ -1525,7 +1525,7 @@ file, You can obtain one at http://mozilla.org/MPL/2.0/.
this.getAttribute("pageproxystate") != "valid")
return;
var urlString = gBrowser.selectedBrowser.currentURI.spec;
var urlString = gBrowser.selectedBrowser.currentURI.displaySpec;
var title = gBrowser.selectedBrowser.contentTitle || urlString;
var htmlString = "<a href=\"" + urlString + "\">" + urlString + "</a>";

4
browser/base/content/utilityOverlay.js
View File

@@ -971,11 +971,11 @@ function trimURL(aURL) {
let fixedUpURL, expectedURLSpec;
try {
fixedUpURL = Services.uriFixup.createFixupURI(urlWithoutProtocol, flags);
expectedURLSpec = makeURI(aURL).spec;
expectedURLSpec = makeURI(aURL).displaySpec;
} catch (ex) {
return url;
}
if (fixedUpURL.spec == expectedURLSpec) {
if (fixedUpURL.displaySpec == expectedURLSpec) {
return urlWithoutProtocol;
}
return url;

2
browser/components/extensions/test/xpcshell/test_ext_bookmarks.js
View File

@@ -240,7 +240,7 @@ add_task(async function test_bookmarks() {
checkOnCreated(results[3].id, bookmarkGuids.unfiledGuid, 0, "EFF", "http://eff.org/", results[3].dateAdded);
checkOnCreated(results[2].id, bookmarkGuids.unfiledGuid, 0, "Mozilla Folder", undefined, results[2].dateAdded);
checkOnCreated(results[1].id, bookmarkGuids.unfiledGuid, 0, "Example", "http://example.org/", results[1].dateAdded);
checkOnCreated(results[0].id, bookmarkGuids.unfiledGuid, 0, "MØzillä", "http://møzîllä.örg/", results[0].dateAdded);
checkOnCreated(results[0].id, bookmarkGuids.unfiledGuid, 0, "MØzillä", "http://xn--mzll-ooa1dud.xn--rg-eka/", results[0].dateAdded);
for (let result of results) {
if (result.title !== "Mozilla Folder") {

7
browser/modules/UpdateTopLevelContentWindowIDHelper.jsm
View File

@@ -78,10 +78,9 @@ function _handleEvent(aEvent) {
function _handleMessage(aMessage) {
let browser = aMessage.target;
if (aMessage.name === "Browser:Init") {
if (browser === browser.ownerGlobal.gBrowser.selectedBrowser) {
_updateCurrentContentOuterWindowID(browser);
}
if (aMessage.name === "Browser:Init" &&
browser === browser.ownerGlobal.gBrowser.selectedBrowser) {
_updateCurrentContentOuterWindowID(browser);
}
}

6
caps/NullPrincipalURI.cpp
View File

@@ -367,6 +367,12 @@ NullPrincipalURI::GetDisplayHost(nsACString &aUnicodeHost)
return GetHost(aUnicodeHost);
}
NS_IMETHODIMP
NullPrincipalURI::GetDisplayPrePath(nsACString &aPrePath)
{
return GetPrePath(aPrePath);
}
////////////////////////////////////////////////////////////////////////////////
//// nsIIPCSerializableURI

4
docshell/test/unit/test_nsDefaultURIFixup_info.js
View File

@@ -337,8 +337,8 @@ var testcases = [ {
inWhitelist: true,
}, {
input: "café.local",
fixedURI: "http://café.local/",
alternateURI: "http://www.café.local/",
fixedURI: "http://xn--caf-dma.local/",
alternateURI: "http://www.xn--caf-dma.local/",
protocolChange: true
}, {
input: "47.6182,-122.830",

1
dom/base/test/chrome/chrome.ini
View File

@@ -70,6 +70,7 @@ support-files = ../file_bug357450.js
[test_nsITextInputProcessor.xul]
[test_range_getClientRectsAndTexts.html]
[test_title.xul]
support-files = file_title.xul
[test_windowroot.xul]
[test_swapFrameLoaders.xul]
[test_groupedSHistory.xul]

1
dom/base/test/chrome/file_title.xul Normal file
View File

@@ -0,0 +1 @@
<window xmlns='http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul' title='Test'/>

4
dom/base/test/chrome/title_window.xul
View File

@@ -13,8 +13,8 @@
<iframe type="content" id="xhtml4" src="data:text/xml,&lt;html xmlns='http://www.w3.org/1999/xhtml'/>"/>
<iframe type="content" id="xhtml5" src="data:text/xml,&lt;html xmlns='http://www.w3.org/1999/xhtml'&gt;&lt;head/>&lt;/html&gt;"/>
<iframe type="content" id="xhtml6" src="data:text/xml,&lt;html xmlns='http://www.w3.org/1999/xhtml'&gt;&lt;head&gt;&lt;style/>&lt;/head&gt;&lt;/html&gt;"/>
<iframe id="xul1" src="data:application/vnd.mozilla.xul+xml,&lt;window xmlns='http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul' title='Test'/>"/>
<iframe id="xul2" src="data:application/vnd.mozilla.xul+xml,&lt;window xmlns='http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul' title='Test'/>"/>
<iframe id="xul1" src="file_title.xul"/>
<iframe id="xul2" src="file_title.xul"/>
<iframe type="content" id="svg1" src="data:text/xml,&lt;svg xmlns='http://www.w3.org/2000/svg'&gt;&lt;title id='t'&gt;Test&lt;/title&gt;&lt;/svg&gt;"/>
<iframe type="content" id="svg2" src="data:text/xml,&lt;svg xmlns='http://www.w3.org/2000/svg'&gt;&lt;title id='t'&gt;Test&lt;/title&gt;&lt;/svg&gt;"/>

1
dom/base/test/file_title.xul Normal file
View File

@@ -0,0 +1 @@
<window xmlns='http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul' title='Test'/>

1
dom/base/test/mochitest.ini
View File

@@ -780,6 +780,7 @@ skip-if = debug == false
skip-if = debug == true && toolkit == 'android' # Timing dependent, skip slow debug android builds
[test_timer_flood.html]
[test_title.html]
support-files = file_title.xul
[test_treewalker_nextsibling.xml]
[test_user_select.html]
skip-if = toolkit == 'android'

9
dom/base/test/test_title.html
View File

@@ -22,19 +22,20 @@
<iframe id="xhtml4" src="data:text/xml,<html xmlns='http://www.w3.org/1999/xhtml'/>"></iframe>
<iframe id="xhtml5" src="data:text/xml,<html xmlns='http://www.w3.org/1999/xhtml'><head/></html>"></iframe>
<iframe id="xhtml6" src="data:text/xml,<html xmlns='http://www.w3.org/1999/xhtml'><head><style/></head></html>"></iframe>
<iframe id="xul1" src="data:application/vnd.mozilla.xul+xml,<window xmlns='http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul' title='Test'/>"></iframe>
<iframe id="xul2" src="data:application/vnd.mozilla.xul+xml,<window xmlns='http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul' title='Test'/>"></iframe>
<iframe id="xul1" src="file_title.xul"></iframe>
<iframe id="xul2" src="file_title.xul"></iframe>
<iframe id="svg1" src="data:text/xml,<svg xmlns='http://www.w3.org/2000/svg'><title id='t'>Test</title></svg>"></iframe>
<iframe id="svg2" src="data:text/xml,<svg xmlns='http://www.w3.org/2000/svg'><title id='t'>Test</title></svg>"></iframe>
</div>
<pre id="test">
<script>
SimpleTest.waitForExplicitFinish();
function runTests() {
function testStatic(id, expect, description) {
is(document.getElementById(id).contentDocument.title, expect, description);
var myFrame = document.getElementById(id);
var wrappedDoc = SpecialPowers.wrap(myFrame).contentDocument;
is(wrappedDoc.title, expect, description);
}
testStatic("html1", "Test", "HTML <title>");

13
dom/media/test/crashtests/1388372.html Normal file
View File

@@ -0,0 +1,13 @@
<!DOCTYPE html>
<html class="reftest-wait">
<script>
navigator.mediaDevices.getUserMedia({audio: {
echoCancellation: false,
noiseSuppression: false,
autoGainControl: false
}, fake: true
}).then((stream) => {
document.documentElement.removeAttribute("class");
})
</script>
</html>

1
dom/media/test/crashtests/crashtests.list
View File

@@ -101,3 +101,4 @@ skip-if(Android&&AndroidVersion=='22') load video-replay-after-audio-end.html #
# This needs to run at the end to avoid leaking busted state into other tests.
load 691096-1.html
load 1236639.html
test-pref(media.navigator.permission.disabled,true) load 1388372.html

5
dom/media/webrtc/MediaEngineWebRTCAudio.cpp
View File

@@ -487,8 +487,9 @@ MediaEngineWebRTCMicrophoneSource::Start(SourceMediaStream *aStream,
// Make sure logger starts before capture
AsyncLatencyLogger::Get(true);
MOZ_ASSERT(mAudioOutputObserver);
mAudioOutputObserver->Clear();
if (mAudioOutputObserver) {
mAudioOutputObserver->Clear();
}
if (mVoEBase->StartReceive(mChannel)) {
return NS_ERROR_FAILURE;

4
dom/push/test/xpcshell/test_record.js
View File

@@ -82,9 +82,9 @@ add_task(async function test_principal_permissions() {
scope: 'https://example.com/',
originAttributes: '^userContextId=1',
}, {
scope: 'https://блог.фанфрог.рф/',
scope: 'https://xn--90aexm.xn--80ag3aejvc.xn--p1ai/',
}, {
scope: 'https://блог.фанфрог.рф/',
scope: 'https://xn--90aexm.xn--80ag3aejvc.xn--p1ai/',
originAttributes: '^userContextId=1',
}];
for (let props of testProps) {

2
dom/tests/mochitest/dom-level0/idn_child.html
View File

@@ -29,7 +29,7 @@ function receiveMessage(evt)
}
else
{
if (domain !== "sub1.παράδειγμα.δοκιμή")
if (domain !== "sub1.xn--hxajbheg2az3al.xn--jxalpdlp")
message += " wrong-initial-domain(" + domain + ")";
}

6
dom/tests/mochitest/dom-level0/test_setting_document.domain_idn.html
View File

@@ -14,6 +14,8 @@
<!--
TODO: after bug 945240 this test is no longer relevant. Should fix.
This testing all gets a bit complicated here; the problem is that our
document.domain implementation will do a suffix comparison of the value to which
it's being set against the current URI's base domain (where "base domain" is
@@ -106,9 +108,7 @@ function receiveMessage(evt)
}
else
{
// We're receiving data from the Greek IDN name; since that TLD is
// whitelisted for now, the domain we get isn't going to be punycoded.
is(evt.origin, "http://sub1.παράδειγμα.δοκιμή", "wrong sender");
is(evt.origin, "http://sub1.xn--hxajbheg2az3al.xn--jxalpdlp", "wrong sender");
}
is(messages[state] + "-response", evt.data.split(" ")[0],

2
dom/tests/mochitest/whatwg/test_postMessage_idn.xhtml
View File

@@ -36,7 +36,7 @@ function receiveMessage(evt)
ok(evt.isTrusted === false, "shouldn't have been a trusted event");
}
is(evt.origin, "http://sub1.ält.example.org:8000",
is(evt.origin, "http://sub1.xn--lt-uia.example.org:8000",
"wrong origin -- IDN issue, perhaps?");
is(evt.data, "idn-response", "unexpected test result");

58
dom/tests/mochitest/whatwg/test_postMessage_origin.xhtml
View File

@@ -268,120 +268,100 @@ var tests =
{
args: ["PASS", "http://sub1.παράδειγμα.δοκιμή"],
source: "idnKidWhitelist",
returnOrigin: "http://sub1.παράδειγμα.δοκιμή"
returnOrigin: "http://sub1.xn--hxajbheg2az3al.xn--jxalpdlp"
},
{
args: ["PASS", "http://sub1.παράδειγμα.δοκιμή:80"],
source: "idnKidWhitelist",
returnOrigin: "http://sub1.παράδειγμα.δοκιμή"
returnOrigin: "http://sub1.xn--hxajbheg2az3al.xn--jxalpdlp"
},
{
args: ["PASS", "http://sub1.παράδειγμα.δοκιμή:80/"],
source: "idnKidWhitelist",
returnOrigin: "http://sub1.παράδειγμα.δοκιμή"
returnOrigin: "http://sub1.xn--hxajbheg2az3al.xn--jxalpdlp"
},
{
args: ["PASS", "http://sub1.παράδειγμα.δοκιμή:80/foobar"],
source: "idnKidWhitelist",
returnOrigin: "http://sub1.παράδειγμα.δοκιμή"
returnOrigin: "http://sub1.xn--hxajbheg2az3al.xn--jxalpdlp"
},
// 40
{
args: ["PASS", "http://sub1.παράδειγμα.δοκιμή/foobar"],
source: "idnKidWhitelist",
returnOrigin: "http://sub1.παράδειγμα.δοκιμή"
returnOrigin: "http://sub1.xn--hxajbheg2az3al.xn--jxalpdlp"
},
{
args: ["PASS", "http://sub1.xn--hxajbheg2az3al.xn--jxalpdlp"],
source: "idnKidWhitelist",
returnOrigin: "http://sub1.παράδειγμα.δοκιμή"
returnOrigin: "http://sub1.xn--hxajbheg2az3al.xn--jxalpdlp"
},
{
args: ["PASS", "http://sub1.xn--hxajbheg2az3al.xn--jxalpdlp:80"],
source: "idnKidWhitelist",
returnOrigin: "http://sub1.παράδειγμα.δοκιμή"
returnOrigin: "http://sub1.xn--hxajbheg2az3al.xn--jxalpdlp"
},
{
args: ["PASS", "http://sub1.xn--hxajbheg2az3al.xn--jxalpdlp:80/"],
source: "idnKidWhitelist",
returnOrigin: "http://sub1.παράδειγμα.δοκιμή"
returnOrigin: "http://sub1.xn--hxajbheg2az3al.xn--jxalpdlp"
},
{
args: ["PASS", "http://sub1.xn--hxajbheg2az3al.xn--jxalpdlp:80/foo"],
source: "idnKidWhitelist",
returnOrigin: "http://sub1.παράδειγμα.δοκιμή"
returnOrigin: "http://sub1.xn--hxajbheg2az3al.xn--jxalpdlp"
},
// 45
{
args: ["PASS", "http://sub1.exaмple.test"],
source: "idnKidNoWhitelist",
returnOrigin: "http://sub1.exaмple.test",
hasWrongReturnOriginBug: true
returnOrigin: "http://sub1.xn--exaple-kqf.test",
},
{
args: ["PASS", "http://sub1.exaмple.test:80"],
source: "idnKidNoWhitelist",
returnOrigin: "http://sub1.exaмple.test",
hasWrongReturnOriginBug: true
returnOrigin: "http://sub1.xn--exaple-kqf.test",
},
{
args: ["PASS", "http://sub1.exaмple.test:80/"],
source: "idnKidNoWhitelist",
returnOrigin: "http://sub1.exaмple.test",
hasWrongReturnOriginBug: true
returnOrigin: "http://sub1.xn--exaple-kqf.test",
},
{
args: ["PASS", "http://sub1.exaмple.test/"],
source: "idnKidNoWhitelist",
returnOrigin: "http://sub1.exaмple.test",
hasWrongReturnOriginBug: true
returnOrigin: "http://sub1.xn--exaple-kqf.test",
},
{
args: ["PASS", "http://sub1.exaмple.test/foobar"],
source: "idnKidNoWhitelist",
returnOrigin: "http://sub1.exaмple.test",
hasWrongReturnOriginBug: true
returnOrigin: "http://sub1.xn--exaple-kqf.test",
},
// 50
{
args: ["PASS", "http://sub1.xn--exaple-kqf.test"],
source: "idnKidNoWhitelist",
returnOrigin: "http://sub1.exaмple.test",
hasWrongReturnOriginBug: true
returnOrigin: "http://sub1.xn--exaple-kqf.test",
},
{
args: ["PASS", "http://sub1.xn--exaple-kqf.test:80"],
source: "idnKidNoWhitelist",
returnOrigin: "http://sub1.exaмple.test",
hasWrongReturnOriginBug: true
returnOrigin: "http://sub1.xn--exaple-kqf.test",
},
{
args: ["PASS", "http://sub1.xn--exaple-kqf.test:80/"],
source: "idnKidNoWhitelist",
returnOrigin: "http://sub1.exaмple.test",
hasWrongReturnOriginBug: true
returnOrigin: "http://sub1.xn--exaple-kqf.test",
},
{
args: ["PASS", "http://sub1.xn--exaple-kqf.test/"],
source: "idnKidNoWhitelist",
returnOrigin: "http://sub1.exaмple.test",
hasWrongReturnOriginBug: true
returnOrigin: "http://sub1.xn--exaple-kqf.test",
},
{
args: ["PASS", "http://sub1.xn--exaple-kqf.test/foobar"],
source: "idnKidNoWhitelist",
returnOrigin: "http://sub1.exaмple.test",
hasWrongReturnOriginBug: true
returnOrigin: "http://sub1.xn--exaple-kqf.test",
},
// 55
{

31
dom/url/tests/test_url.html
View File

@@ -227,6 +227,37 @@
hash: '#yeah',
skip_setters: false,
},
{ url: 'http://sub2.xn--lt-uia.mochi.test:8888/foo',
base: undefined,
error: false,
href: 'http://sub2.xn--lt-uia.mochi.test:8888/foo',
origin: 'http://sub2.xn--lt-uia.mochi.test:8888',
protocol: 'http:',
username: '',
password: '',
host: 'sub2.xn--lt-uia.mochi.test:8888',
hostname: 'sub2.xn--lt-uia.mochi.test',
port: '8888',
pathname: '/foo',
search: '',
hash: ''
},
{ url: 'http://sub2.ält.mochi.test:8888/foo',
base: undefined,
error: false,
href: 'http://sub2.xn--lt-uia.mochi.test:8888/foo',
origin: 'http://sub2.xn--lt-uia.mochi.test:8888',
protocol: 'http:',
username: '',
password: '',
host: 'sub2.xn--lt-uia.mochi.test:8888',
hostname: 'sub2.xn--lt-uia.mochi.test',
port: '8888',
pathname: '/foo',
search: '',
hash: ''
},
];
while(tests.length) {

9
gfx/harfbuzz/NEWS
View File

@@ -1,3 +1,12 @@
Overview of changes leading to 1.4.8
Tuesday, August 8, 2017
====================================
- Major fix to avar table handling.
- Rename hb-shape --show-message to --trace.
- Build fixes.
Overview of changes leading to 1.4.7
Tuesday, July 18, 2017
====================================

4
gfx/harfbuzz/README-mozilla
View File

@@ -1,9 +1,9 @@
gfx/harfbuzz status as of 2017-07-19:
gfx/harfbuzz status as of 2017-08-08:
This directory contains the harfbuzz source from the 'master' branch of
https://github.com/behdad/harfbuzz.
Current version: 1.4.7
Current version: 1.4.8
UPDATING:

7
gfx/harfbuzz/THANKS Normal file
View File

@@ -0,0 +1,7 @@
Bradley Grainger
Khaled Hosny
Kenichi Ishibashi
Ryan Lortie
Jeff Muizelaar
suzuki toshiya
Philip Withnall

2
gfx/harfbuzz/configure.ac
View File

@@ -1,6 +1,6 @@
AC_PREREQ([2.64])
AC_INIT([HarfBuzz],
[1.4.7],
[1.4.8],
[https://github.com/behdad/harfbuzz/issues/new],
[harfbuzz],
[http://harfbuzz.org/])

2
gfx/harfbuzz/src/check-defs.sh
View File

@@ -21,7 +21,7 @@ for def in $defs; do
lib=`echo "$def" | sed 's/[.]def$//;s@.*/@@'`
so=.libs/lib${lib}.so
EXPORTED_SYMBOLS="`nm "$so" | grep ' [BCDGINRSTVW] ' | grep -v ' _fini\>\| _init\>\| _fdata\>\| _ftext\>\| _fbss\>\| __bss_start\>\| __bss_start__\>\| __bss_end__\>\| _edata\>\| _end\>\| _bss_end__\>\| __end__\>\| __gcov_flush\>\| llvm_' | cut -d' ' -f3`"
EXPORTED_SYMBOLS="`nm "$so" | grep ' [BCDGINRSTVW] .' | grep -v ' _fini\>\| _init\>\| _fdata\>\| _ftext\>\| _fbss\>\| __bss_start\>\| __bss_start__\>\| __bss_end__\>\| _edata\>\| _end\>\| _bss_end__\>\| __end__\>\| __gcov_flush\>\| llvm_' | cut -d' ' -f3`"
if test -f "$so"; then

2
gfx/harfbuzz/src/check-symbols.sh
View File

@@ -20,7 +20,7 @@ for suffix in so dylib; do
so=.libs/libharfbuzz.$suffix
if ! test -f "$so"; then continue; fi
EXPORTED_SYMBOLS="`nm "$so" | grep ' [BCDGINRSTVW] ' | grep -v ' _fini\>\| _init\>\| _fdata\>\| _ftext\>\| _fbss\>\| __bss_start\>\| __bss_start__\>\| __bss_end__\>\| _edata\>\| _end\>\| _bss_end__\>\| __end__\>\| __gcov_flush\>\| ___gcov_flush\>\| llvm_\| _llvm_' | cut -d' ' -f3`"
EXPORTED_SYMBOLS="`nm "$so" | grep ' [BCDGINRSTVW] .' | grep -v ' _fini\>\| _init\>\| _fdata\>\| _ftext\>\| _fbss\>\| __bss_start\>\| __bss_start__\>\| __bss_end__\>\| _edata\>\| _end\>\| _bss_end__\>\| __end__\>\| __gcov_flush\>\| ___gcov_flush\>\| llvm_\| _llvm_' | cut -d' ' -f3`"
prefix=`basename "$so" | sed 's/libharfbuzz/hb/; s/-/_/g; s/[.].*//'`

2
gfx/harfbuzz/src/harfbuzz-icu.pc
View File

@@ -5,7 +5,7 @@ includedir=/usr/local/include
Name: harfbuzz
Description: HarfBuzz text shaping library ICU integration
Version: 1.4.7
Version: 1.4.8
Requires: harfbuzz
Requires.private: icu-uc

2
gfx/harfbuzz/src/harfbuzz.pc
View File

@@ -5,7 +5,7 @@ includedir=/usr/local/include
Name: harfbuzz
Description: HarfBuzz text shaping library
Version: 1.4.7
Version: 1.4.8
Libs: -L${libdir} -lharfbuzz
Libs.private:

2
gfx/harfbuzz/src/hb-ot-shape-complex-indic.cc
View File

@@ -200,7 +200,7 @@ set_indic_properties (hb_glyph_info_t &info)
cat = OT_Symbol;
ASSERT_STATIC ((int) INDIC_SYLLABIC_CATEGORY_AVAGRAHA == OT_Symbol);
}
else if (unlikely (hb_in_range (u, 0x17CDu, 0x17D1u) ||
else if (unlikely (hb_in_range<hb_codepoint_t> (u, 0x17CDu, 0x17D1u) ||
u == 0x17CBu || u == 0x17D3u || u == 0x17DDu)) /* Khmer Various signs */
{
/* These can occur mid-syllable (eg. before matras), even though Unicode marks them as Syllable_Modifier.

13
gfx/harfbuzz/src/hb-ot-var-avar-table.hh
View File

@@ -57,8 +57,13 @@ struct SegmentMaps : ArrayOf<AxisValueMap>
* that at least -1, 0, and +1 must be mapped. But we include these as
* part of a better error recovery scheme. */
if (!len)
return value;
if (len < 2)
{
if (!len)
return value;
else /* len == 1*/
return value - array[0].fromCoord + array[0].toCoord;
}
if (value <= array[0].fromCoord)
return value - array[0].fromCoord + array[0].toCoord;
@@ -76,8 +81,8 @@ struct SegmentMaps : ArrayOf<AxisValueMap>
int denom = array[i].fromCoord - array[i-1].fromCoord;
return array[i-1].toCoord +
(array[i].toCoord - array[i-1].toCoord) *
(value - array[i-1].fromCoord + denom/2) / denom;
((array[i].toCoord - array[i-1].toCoord) *
(value - array[i-1].fromCoord) + denom/2) / denom;
}
DEFINE_SIZE_ARRAY (2, array);

4
gfx/harfbuzz/src/hb-version.h
View File

@@ -38,9 +38,9 @@ HB_BEGIN_DECLS
#define HB_VERSION_MAJOR 1
#define HB_VERSION_MINOR 4
#define HB_VERSION_MICRO 7
#define HB_VERSION_MICRO 8
#define HB_VERSION_STRING "1.4.7"
#define HB_VERSION_STRING "1.4.8"
#define HB_VERSION_ATLEAST(major,minor,micro) \
((major)*10000+(minor)*100+(micro) <= \

48
gfx/layers/RotatedBuffer.cpp
View File

@@ -314,6 +314,7 @@ RotatedContentBuffer::BorrowDrawTargetForQuadrantUpdate(const IntRect& aBounds,
MOZ_ASSERT(aOutMatrix);
*aOutMatrix = transform;
}
return mLoanedDrawTarget;
}
@@ -751,17 +752,19 @@ RotatedContentBuffer::BorrowDrawTargetForRecording(PaintState& aPaintState,
Matrix transform;
DrawTarget* result = BorrowDrawTargetForQuadrantUpdate(aPaintState.mRegionToDraw.GetBounds(),
BUFFER_BOTH, aIter,
false, &transform);
false,
&transform);
if (!result) {
return nullptr;
}
ExpandDrawRegion(aPaintState, aIter, result->GetBackendType());
nsIntRegion regionToDraw =
ExpandDrawRegion(aPaintState, aIter, result->GetBackendType());
RefPtr<CapturedPaintState> state =
new CapturedPaintState(aPaintState.mRegionToDraw,
new CapturedPaintState(regionToDraw,
result,
nullptr, /* aTargetOnWhite */
mDTBufferOnWhite,
transform,
aPaintState.mMode,
aPaintState.mContentType);
@@ -771,6 +774,7 @@ RotatedContentBuffer::BorrowDrawTargetForRecording(PaintState& aPaintState,
/*static */ bool
RotatedContentBuffer::PrepareDrawTargetForPainting(CapturedPaintState* aState)
{
MOZ_ASSERT(aState);
RefPtr<DrawTarget> target = aState->mTarget;
RefPtr<DrawTarget> whiteTarget = aState->mTargetOnWhite;
@@ -801,7 +805,7 @@ RotatedContentBuffer::PrepareDrawTargetForPainting(CapturedPaintState* aState)
return true;
}
void
nsIntRegion
RotatedContentBuffer::ExpandDrawRegion(PaintState& aPaintState,
DrawIterator* aIter,
BackendType aBackendType)
@@ -819,41 +823,29 @@ RotatedContentBuffer::ExpandDrawRegion(PaintState& aPaintState,
// for complex regions.
drawPtr->SimplifyOutwardByArea(100 * 100);
}
return *drawPtr;
}
DrawTarget*
RotatedContentBuffer::BorrowDrawTargetForPainting(PaintState& aPaintState,
DrawIterator* aIter /* = nullptr */)
{
if (aPaintState.mMode == SurfaceMode::SURFACE_NONE) {
RefPtr<CapturedPaintState> capturedState =
BorrowDrawTargetForRecording(aPaintState, aIter);
if (!capturedState) {
return nullptr;
}
DrawTarget* result = BorrowDrawTargetForQuadrantUpdate(aPaintState.mRegionToDraw.GetBounds(),
BUFFER_BOTH, aIter);
if (!result) {
// BorrowDrawTargetForRecording doesn't apply the transform, so we have to.
RefPtr<DrawTarget> target = capturedState->mTarget;
target->SetTransform(capturedState->mTargetTransform);
if (!RotatedContentBuffer::PrepareDrawTargetForPainting(capturedState)) {
return nullptr;
}
ExpandDrawRegion(aPaintState, aIter, result->GetBackendType());
nsIntRegion regionToDraw = aIter ? aIter->mDrawRegion
: aPaintState.mRegionToDraw;
// Can't stack allocate refcounted objects.
RefPtr<CapturedPaintState> capturedPaintState =
MakeAndAddRef<CapturedPaintState>(regionToDraw,
mDTBuffer,
mDTBufferOnWhite,
Matrix(),
aPaintState.mMode,
aPaintState.mContentType);
if (!RotatedContentBuffer::PrepareDrawTargetForPainting(capturedPaintState)) {
return nullptr;
}
return result;
return target;
}
already_AddRefed<SourceSurface>

6
gfx/layers/RotatedBuffer.h
View File

@@ -305,9 +305,9 @@ public:
RefPtr<CapturedPaintState> BorrowDrawTargetForRecording(PaintState& aPaintState,
DrawIterator* aIter);
void ExpandDrawRegion(PaintState& aPaintState,
DrawIterator* aIter,
gfx::BackendType aBackendType);
nsIntRegion ExpandDrawRegion(PaintState& aPaintState,
DrawIterator* aIter,
gfx::BackendType aBackendType);
static bool PrepareDrawTargetForPainting(CapturedPaintState*);
enum {

6
gfx/layers/client/ClientPaintedLayer.cpp
View File

@@ -71,12 +71,6 @@ ClientPaintedLayer::CanRecordLayer(ReadbackProcessor* aReadback)
return false;
}
// Component alpha layers aren't supported yet since we have to
// hold onto both the front/back buffer of a texture client.
if (GetSurfaceMode() == SurfaceMode::SURFACE_COMPONENT_ALPHA) {
return false;
}
return GetAncestorMaskLayerCount() == 0;
}

7
gfx/layers/d3d11/MLGDeviceD3D11.cpp
View File

@@ -625,7 +625,11 @@ MLGBufferD3D11::Create(ID3D11Device* aDevice,
data.SysMemSlicePitch = 0;
RefPtr<ID3D11Buffer> buffer;
aDevice->CreateBuffer(&desc, aInitialData ? &data : nullptr, getter_AddRefs(buffer));
HRESULT hr = aDevice->CreateBuffer(&desc, aInitialData ? &data : nullptr, getter_AddRefs(buffer));
if (FAILED(hr) || !buffer) {
gfxCriticalError() << "Failed to create ID3D11Buffer.";
return nullptr;
}
return new MLGBufferD3D11(buffer, aType, aSize);
}
@@ -1337,6 +1341,7 @@ bool
MLGDeviceD3D11::Map(MLGResource* aResource, MLGMapType aType, MLGMappedResource* aMap)
{
ID3D11Resource* resource = aResource->AsResourceD3D11()->GetResource();
MOZ_ASSERT(resource);
D3D11_MAPPED_SUBRESOURCE map;
HRESULT hr = mCtx->Map(resource, 0, ToD3D11Map(aType), 0, &map);

14
gfx/thebes/gfxContext.cpp
View File

@@ -610,20 +610,6 @@ gfxContext::GetClipExtents()
return ThebesRect(rect);
}
bool
gfxContext::HasComplexClip() const
{
for (int i = mStateStack.Length() - 1; i >= 0; i--) {
for (unsigned int c = 0; c < mStateStack[i].pushedClips.Length(); c++) {
const AzureState::PushedClip &clip = mStateStack[i].pushedClips[c];
if (clip.path || !clip.transform.IsRectilinear()) {
return true;
}
}
}
return false;
}
bool
gfxContext::ExportClip(ClipExporter& aExporter)
{

5
gfx/thebes/gfxContext.h
View File

@@ -383,11 +383,6 @@ public:
*/
gfxRect GetClipExtents();
/**
* Whether the current clip is not a simple rectangle.
*/
bool HasComplexClip() const;
/**
* Returns true if the given rectangle is fully contained in the current clip.
* This is conservative; it may return false even when the given rectangle is

31
image/SurfacePipe.cpp
View File

@@ -6,10 +6,8 @@
#include "SurfacePipe.h"
#include <utility>
#include <algorithm> // for min
#include "mozilla/ClearOnShutdown.h"
#include "mozilla/DebugOnly.h"
#include "Decoder.h"
namespace mozilla {
@@ -19,33 +17,6 @@ using namespace gfx;
using std::min;
/* static */ UniquePtr<NullSurfaceSink> NullSurfaceSink::sSingleton;
/* static */ NullSurfaceSink*
NullSurfaceSink::Singleton()
{
if (!sSingleton) {
MOZ_ASSERT(NS_IsMainThread());
sSingleton = MakeUnique<NullSurfaceSink>();
ClearOnShutdown(&sSingleton);
DebugOnly<nsresult> rv = sSingleton->Configure(NullSurfaceConfig { });
MOZ_ASSERT(NS_SUCCEEDED(rv), "Couldn't configure a NullSurfaceSink?");
}
return sSingleton.get();
}
nsresult
NullSurfaceSink::Configure(const NullSurfaceConfig& aConfig)
{
// Note that the choice of uint32_t as the pixel size here is more or less
// arbitrary, since you cannot write to a NullSurfaceSink anyway, but uint32_t
// is a natural choice since most SurfacePipes will be for BGRA/BGRX surfaces.
ConfigureFilter(IntSize(), sizeof(uint32_t));
return NS_OK;
}
Maybe<SurfaceInvalidRect>
AbstractSurfaceSink::TakeInvalidRect()
{

67
image/SurfacePipe.h
View File

@@ -514,44 +514,6 @@ private:
uint8_t mPixelSize; /// How large each pixel in the surface is, in bytes.
};
class NullSurfaceSink;
/// A trivial configuration struct for NullSurfaceSink.
struct NullSurfaceConfig
{
using Filter = NullSurfaceSink;
};
/**
* NullSurfaceSink is a trivial SurfaceFilter implementation that behaves as if
* it were a zero-size SurfaceSink. It's used as the default filter chain for an
* uninitialized SurfacePipe.
*
* To avoid unnecessary allocations when creating SurfacePipe objects,
* NullSurfaceSink is a singleton. (This implies that the implementation must be
* stateless.)
*/
class NullSurfaceSink final : public SurfaceFilter
{
public:
/// Returns the singleton instance of NullSurfaceSink.
static NullSurfaceSink* Singleton();
virtual ~NullSurfaceSink() { }
nsresult Configure(const NullSurfaceConfig& aConfig);
Maybe<SurfaceInvalidRect> TakeInvalidRect() override { return Nothing(); }
protected:
uint8_t* DoResetToFirstRow() override { return nullptr; }
uint8_t* DoAdvanceRow() override { return nullptr; }
private:
static UniquePtr<NullSurfaceSink> sSingleton; /// The singleton instance.
};
/**
* SurfacePipe is the public API that decoders should use to interact with a
* SurfaceFilter pipeline.
@@ -559,11 +521,7 @@ private:
class SurfacePipe
{
public:
/// Initialize global state used by all SurfacePipes.
static void Initialize() { NullSurfaceSink::Singleton(); }
SurfacePipe()
: mHead(NullSurfaceSink::Singleton())
{ }
SurfacePipe(SurfacePipe&& aOther)
@@ -571,28 +529,21 @@ public:
{ }
~SurfacePipe()
{
// Ensure that we don't free the NullSurfaceSink singleton.
if (mHead.get() == NullSurfaceSink::Singleton()) {
Unused << mHead.release();
}
}
{ }
SurfacePipe& operator=(SurfacePipe&& aOther)
{
MOZ_ASSERT(this != &aOther);
// Ensure that we don't free the NullSurfaceSink singleton.
if (mHead.get() == NullSurfaceSink::Singleton()) {
Unused << mHead.release();
}
mHead = Move(aOther.mHead);
return *this;
}
/// Begins a new pass, seeking to the first row of the surface.
void ResetToFirstRow() { mHead->ResetToFirstRow(); }
void ResetToFirstRow()
{
MOZ_ASSERT(mHead, "Use before configured!");
mHead->ResetToFirstRow();
}
/**
* Write pixels to the surface one at a time by repeatedly calling a lambda
@@ -603,6 +554,7 @@ public:
template <typename PixelType, typename Func>
WriteState WritePixels(Func aFunc)
{
MOZ_ASSERT(mHead, "Use before configured!");
return mHead->WritePixels<PixelType>(Forward<Func>(aFunc));
}
@@ -616,6 +568,7 @@ public:
template <typename PixelType, typename Func>
WriteState WritePixelsToRow(Func aFunc)
{
MOZ_ASSERT(mHead, "Use before configured!");
return mHead->WritePixelsToRow<PixelType>(Forward<Func>(aFunc));
}
@@ -631,6 +584,7 @@ public:
template <typename PixelType>
WriteState WriteBuffer(const PixelType* aSource)
{
MOZ_ASSERT(mHead, "Use before configured!");
return mHead->WriteBuffer<PixelType>(aSource);
}
@@ -649,6 +603,7 @@ public:
const size_t aStartColumn,
const size_t aLength)
{
MOZ_ASSERT(mHead, "Use before configured!");
return mHead->WriteBuffer<PixelType>(aSource, aStartColumn, aLength);
}
@@ -660,6 +615,7 @@ public:
*/
WriteState WriteEmptyRow()
{
MOZ_ASSERT(mHead, "Use before configured!");
return mHead->WriteEmptyRow();
}
@@ -669,6 +625,7 @@ public:
/// @see SurfaceFilter::TakeInvalidRect() for the canonical documentation.
Maybe<SurfaceInvalidRect> TakeInvalidRect() const
{
MOZ_ASSERT(mHead, "Use before configured!");
return mHead->TakeInvalidRect();
}

1
image/build/nsImageModule.cpp
View File

@@ -103,7 +103,6 @@ mozilla::image::EnsureModuleInitialized()
mozilla::image::ImageFactory::Initialize();
mozilla::image::DecodePool::Initialize();
mozilla::image::SurfaceCache::Initialize();
mozilla::image::SurfacePipe::Initialize();
imgLoader::GlobalInit();
sInitialized = true;
return NS_OK;

6
image/decoders/icon/nsIconURI.cpp
View File

@@ -135,6 +135,12 @@ nsMozIconURI::GetDisplayHost(nsACString& aUnicodeHost)
return GetHost(aUnicodeHost);
}
NS_IMETHODIMP
nsMozIconURI::GetDisplayPrePath(nsACString& aPrePath)
{
return GetPrePath(aPrePath);
}
NS_IMETHODIMP
nsMozIconURI::GetHasRef(bool* result)
{

65
image/test/gtest/TestSurfaceSink.cpp
View File

@@ -113,71 +113,6 @@ CheckPalettedIterativeWrite(Decoder* aDecoder,
});
}
TEST(ImageSurfaceSink, NullSurfaceSink)
{
// Create the NullSurfaceSink.
NullSurfaceSink sink;
nsresult rv = sink.Configure(NullSurfaceConfig { });
ASSERT_TRUE(NS_SUCCEEDED(rv));
EXPECT_TRUE(!sink.IsValidPalettedPipe());
// Ensure that we can't write anything.
bool gotCalled = false;
auto result = sink.WritePixels<uint32_t>([&]() {
gotCalled = true;
return AsVariant(BGRAColor::Green().AsPixel());
});
EXPECT_FALSE(gotCalled);
EXPECT_EQ(WriteState::FINISHED, result);
EXPECT_TRUE(sink.IsSurfaceFinished());
Maybe<SurfaceInvalidRect> invalidRect = sink.TakeInvalidRect();
EXPECT_TRUE(invalidRect.isNothing());
uint32_t source = BGRAColor::Red().AsPixel();
result = sink.WriteBuffer(&source);
EXPECT_EQ(WriteState::FINISHED, result);
EXPECT_TRUE(sink.IsSurfaceFinished());
invalidRect = sink.TakeInvalidRect();
EXPECT_TRUE(invalidRect.isNothing());
result = sink.WriteBuffer(&source, 0, 1);
EXPECT_EQ(WriteState::FINISHED, result);
EXPECT_TRUE(sink.IsSurfaceFinished());
invalidRect = sink.TakeInvalidRect();
EXPECT_TRUE(invalidRect.isNothing());
result = sink.WriteEmptyRow();
EXPECT_EQ(WriteState::FINISHED, result);
EXPECT_TRUE(sink.IsSurfaceFinished());
invalidRect = sink.TakeInvalidRect();
EXPECT_TRUE(invalidRect.isNothing());
result = sink.WriteUnsafeComputedRow<uint32_t>([&](uint32_t* aRow,
uint32_t aLength) {
gotCalled = true;
for (uint32_t col = 0; col < aLength; ++col, ++aRow) {
*aRow = BGRAColor::Red().AsPixel();
}
});
EXPECT_FALSE(gotCalled);
EXPECT_EQ(WriteState::FINISHED, result);
EXPECT_TRUE(sink.IsSurfaceFinished());
invalidRect = sink.TakeInvalidRect();
EXPECT_TRUE(invalidRect.isNothing());
// Attempt to advance to the next row and make sure nothing changes.
sink.AdvanceRow();
EXPECT_TRUE(sink.IsSurfaceFinished());
invalidRect = sink.TakeInvalidRect();
EXPECT_TRUE(invalidRect.isNothing());
// Attempt to advance to the next pass and make sure nothing changes.
sink.ResetToFirstRow();
EXPECT_TRUE(sink.IsSurfaceFinished());
invalidRect = sink.TakeInvalidRect();
EXPECT_TRUE(invalidRect.isNothing());
}
TEST(ImageSurfaceSink, SurfaceSinkInitialization)
{
WithSurfaceSink<Orient::NORMAL>([](Decoder* aDecoder, SurfaceSink* aSink) {

86
js/src/builtin/RegExp.cpp
View File

@@ -22,6 +22,7 @@
#include "jsobjinlines.h"
#include "vm/NativeObject-inl.h"
#include "vm/UnboxedObject-inl.h"
using namespace js;
using namespace js::unicode;
@@ -860,7 +861,7 @@ const JSPropertySpec js::regexp_static_props[] = {
template <typename CharT>
static bool
IsTrailSurrogateWithLeadSurrogateImpl(JSContext* cx, HandleLinearString input, size_t index)
IsTrailSurrogateWithLeadSurrogateImpl(HandleLinearString input, size_t index)
{
JS::AutoCheckCannotGC nogc;
MOZ_ASSERT(index > 0 && index < input->length());
@@ -871,14 +872,14 @@ IsTrailSurrogateWithLeadSurrogateImpl(JSContext* cx, HandleLinearString input, s
}
static bool
IsTrailSurrogateWithLeadSurrogate(JSContext* cx, HandleLinearString input, int32_t index)
IsTrailSurrogateWithLeadSurrogate(HandleLinearString input, int32_t index)
{
if (index <= 0 || size_t(index) >= input->length())
return false;
return input->hasLatin1Chars()
? IsTrailSurrogateWithLeadSurrogateImpl<Latin1Char>(cx, input, index)
: IsTrailSurrogateWithLeadSurrogateImpl<char16_t>(cx, input, index);
? IsTrailSurrogateWithLeadSurrogateImpl<Latin1Char>(input, index)
: IsTrailSurrogateWithLeadSurrogateImpl<char16_t>(input, index);
}
/*
@@ -897,7 +898,7 @@ ExecuteRegExp(JSContext* cx, HandleObject regexp, HandleString string,
*/
/* Steps 1-2 performed by the caller. */
Rooted<RegExpObject*> reobj(cx, &regexp->as<RegExpObject>());
Handle<RegExpObject*> reobj = regexp.as<RegExpObject>();
RootedRegExpShared re(cx, RegExpObject::getShared(cx, reobj));
if (!re)
@@ -945,7 +946,7 @@ ExecuteRegExp(JSContext* cx, HandleObject regexp, HandleString string,
* However, the spec will change to match our implementation's
* behavior. See https://github.com/tc39/ecma262/issues/128.
*/
if (IsTrailSurrogateWithLeadSurrogate(cx, input, lastIndex))
if (IsTrailSurrogateWithLeadSurrogate(input, lastIndex))
lastIndex--;
}
@@ -1203,6 +1204,8 @@ js::regexp_test_no_statics(JSContext* cx, unsigned argc, Value* vp)
return status != RegExpRunStatus_Error;
}
using CapturesVector = GCVector<Value, 4>;
static void
GetParen(JSLinearString* matched, const JS::Value& capture, JSSubString* out)
{
@@ -1217,7 +1220,7 @@ GetParen(JSLinearString* matched, const JS::Value& capture, JSSubString* out)
template <typename CharT>
static bool
InterpretDollar(JSLinearString* matched, JSLinearString* string, size_t position, size_t tailPos,
MutableHandle<GCVector<Value>> captures, JSLinearString* replacement,
Handle<CapturesVector> captures, JSLinearString* replacement,
const CharT* replacementBegin, const CharT* currentDollar,
const CharT* replacementEnd,
JSSubString* out, size_t* skip)
@@ -1229,7 +1232,7 @@ InterpretDollar(JSLinearString* matched, JSLinearString* string, size_t position
return false;
/* ES 2016 draft Mar 25, 2016 Table 46. */
char16_t c = currentDollar[1];
CharT c = currentDollar[1];
if (JS7_ISDEC(c)) {
/* $n, $nn */
unsigned num = JS7_UNDEC(c);
@@ -1296,7 +1299,7 @@ InterpretDollar(JSLinearString* matched, JSLinearString* string, size_t position
template <typename CharT>
static bool
FindReplaceLengthString(JSContext* cx, HandleLinearString matched, HandleLinearString string,
size_t position, size_t tailPos, MutableHandle<GCVector<Value>> captures,
size_t position, size_t tailPos, Handle<CapturesVector> captures,
HandleLinearString replacement, size_t firstDollarIndex, size_t* sizep)
{
CheckedInt<uint32_t> replen = replacement->length();
@@ -1335,7 +1338,7 @@ FindReplaceLengthString(JSContext* cx, HandleLinearString matched, HandleLinearS
static bool
FindReplaceLength(JSContext* cx, HandleLinearString matched, HandleLinearString string,
size_t position, size_t tailPos, MutableHandle<GCVector<Value>> captures,
size_t position, size_t tailPos, Handle<CapturesVector> captures,
HandleLinearString replacement, size_t firstDollarIndex, size_t* sizep)
{
return replacement->hasLatin1Chars()
@@ -1353,7 +1356,7 @@ FindReplaceLength(JSContext* cx, HandleLinearString matched, HandleLinearString
template <typename CharT>
static void
DoReplace(HandleLinearString matched, HandleLinearString string,
size_t position, size_t tailPos, MutableHandle<GCVector<Value>> captures,
size_t position, size_t tailPos, Handle<CapturesVector> captures,
HandleLinearString replacement, size_t firstDollarIndex, StringBuffer &sb)
{
JS::AutoCheckCannotGC nogc;
@@ -1388,7 +1391,7 @@ DoReplace(HandleLinearString matched, HandleLinearString string,
static bool
NeedTwoBytes(HandleLinearString string, HandleLinearString replacement,
HandleLinearString matched, Handle<GCVector<Value>> captures)
HandleLinearString matched, Handle<CapturesVector> captures)
{
if (string->hasTwoByteChars())
return true;
@@ -1398,7 +1401,7 @@ NeedTwoBytes(HandleLinearString string, HandleLinearString replacement,
return true;
for (size_t i = 0, len = captures.length(); i < len; i++) {
Value capture = captures[i];
const Value& capture = captures[i];
if (capture.isUndefined())
continue;
if (capture.toString()->hasTwoByteChars())
@@ -1410,14 +1413,25 @@ NeedTwoBytes(HandleLinearString string, HandleLinearString replacement,
/* ES 2016 draft Mar 25, 2016 21.1.3.14.1. */
bool
js::RegExpGetSubstitution(JSContext* cx, HandleLinearString matched, HandleLinearString string,
size_t position, HandleObject capturesObj, HandleLinearString replacement,
js::RegExpGetSubstitution(JSContext* cx, HandleObject matchResult, HandleLinearString string,
size_t position, HandleLinearString replacement,
size_t firstDollarIndex, MutableHandleValue rval)
{
MOZ_ASSERT(firstDollarIndex < replacement->length());
MOZ_ASSERT(matchResult->is<ArrayObject>() || matchResult->is<UnboxedArrayObject>());
// Step 1 (skipped).
// Step 10 (reordered).
uint32_t matchResultLength = GetAnyBoxedOrUnboxedArrayLength(matchResult);
MOZ_ASSERT(matchResultLength > 0);
MOZ_ASSERT(matchResultLength == GetAnyBoxedOrUnboxedInitializedLength(matchResult));
const Value& matchedValue = GetAnyBoxedOrUnboxedDenseElement(matchResult, 0);
RootedLinearString matched(cx, matchedValue.toString()->ensureLinear(cx));
if (!matched)
return false;
// Step 2.
size_t matchLength = matched->length();
@@ -1426,28 +1440,21 @@ js::RegExpGetSubstitution(JSContext* cx, HandleLinearString matched, HandleLinea
// Step 6.
MOZ_ASSERT(position <= string->length());
// Step 10 (reordered).
uint32_t nCaptures;
if (!GetLengthProperty(cx, capturesObj, &nCaptures))
return false;
Rooted<GCVector<Value>> captures(cx, GCVector<Value>(cx));
uint32_t nCaptures = matchResultLength - 1;
Rooted<CapturesVector> captures(cx, CapturesVector(cx));
if (!captures.reserve(nCaptures))
return false;
// Step 7.
RootedValue capture(cx);
for (uint32_t i = 0; i < nCaptures; i++) {
if (!GetElement(cx, capturesObj, capturesObj, i, &capture))
return false;
for (uint32_t i = 1; i <= nCaptures; i++) {
const Value& capture = GetAnyBoxedOrUnboxedDenseElement(matchResult, i);
if (capture.isUndefined()) {
captures.infallibleAppend(capture);
continue;
}
MOZ_ASSERT(capture.isString());
RootedLinearString captureLinear(cx, capture.toString()->ensureLinear(cx));
JSLinearString* captureLinear = capture.toString()->ensureLinear(cx);
if (!captureLinear)
return false;
captures.infallibleAppend(StringValue(captureLinear));
@@ -1467,7 +1474,7 @@ js::RegExpGetSubstitution(JSContext* cx, HandleLinearString matched, HandleLinea
// Step 11.
size_t reserveLength;
if (!FindReplaceLength(cx, matched, string, position, tailPos, &captures, replacement,
if (!FindReplaceLength(cx, matched, string, position, tailPos, captures, replacement,
firstDollarIndex, &reserveLength))
{
return false;
@@ -1483,10 +1490,10 @@ js::RegExpGetSubstitution(JSContext* cx, HandleLinearString matched, HandleLinea
return false;
if (replacement->hasLatin1Chars()) {
DoReplace<Latin1Char>(matched, string, position, tailPos, &captures,
DoReplace<Latin1Char>(matched, string, position, tailPos, captures,
replacement, firstDollarIndex, result);
} else {
DoReplace<char16_t>(matched, string, position, tailPos, &captures,
DoReplace<char16_t>(matched, string, position, tailPos, captures,
replacement, firstDollarIndex, result);
}
@@ -1504,7 +1511,7 @@ js::GetFirstDollarIndex(JSContext* cx, unsigned argc, Value* vp)
{
CallArgs args = CallArgsFromVp(argc, vp);
MOZ_ASSERT(args.length() == 1);
RootedString str(cx, args[0].toString());
JSString* str = args[0].toString();
// Should be handled in different path.
MOZ_ASSERT(str->length() != 0);
@@ -1538,11 +1545,11 @@ js::GetFirstDollarIndexRawFlat(JSLinearString* text)
if (text->hasLatin1Chars())
return GetFirstDollarIndexImpl(text->latin1Chars(nogc), len);
return GetFirstDollarIndexImpl(text->twoByteChars(nogc), len);
return GetFirstDollarIndexImpl(text->twoByteChars(nogc), len);
}
bool
js::GetFirstDollarIndexRaw(JSContext* cx, HandleString str, int32_t* index)
js::GetFirstDollarIndexRaw(JSContext* cx, JSString* str, int32_t* index)
{
JSLinearString* text = str->ensureLinear(cx);
if (!text)
@@ -1755,7 +1762,7 @@ js::intrinsic_GetElemBaseForLambda(JSContext* cx, unsigned argc, Value* vp)
/*
* Emulates `b[a]` property access, that is detected in GetElemBaseForLambda.
* It returns the property value only if the property is data property and the
* propety value is a string. Otherwise it returns undefined.
* property value is a string. Otherwise it returns undefined.
*/
bool
js::intrinsic_GetStringDataProperty(JSContext* cx, unsigned argc, Value* vp)
@@ -1766,21 +1773,18 @@ js::intrinsic_GetStringDataProperty(JSContext* cx, unsigned argc, Value* vp)
RootedObject obj(cx, &args[0].toObject());
if (!obj->isNative()) {
// The object is already checked to be native in GetElemBaseForLambda,
// but it can be swapped to the other class that is non-native.
// but it can be swapped to another class that is non-native.
// Return undefined to mark failure to get the property.
args.rval().setUndefined();
return true;
}
RootedNativeObject nobj(cx, &obj->as<NativeObject>());
RootedString name(cx, args[1].toString());
RootedAtom atom(cx, AtomizeString(cx, name));
JSAtom* atom = AtomizeString(cx, args[1].toString());
if (!atom)
return false;
RootedValue v(cx);
if (GetPropertyPure(cx, nobj, AtomToId(atom), v.address()) && v.isString())
Value v;
if (GetPropertyPure(cx, obj, AtomToId(atom), &v) && v.isString())
args.rval().set(v);
else
args.rval().setUndefined();

8
js/src/builtin/RegExp.h
View File

@@ -118,15 +118,15 @@ extern MOZ_MUST_USE bool
RegExpInstanceOptimizableRaw(JSContext* cx, JSObject* obj, JSObject* proto);
extern MOZ_MUST_USE bool
RegExpGetSubstitution(JSContext* cx, HandleLinearString matched, HandleLinearString string,
size_t position, HandleObject capturesObj, HandleLinearString replacement,
size_t firstDollarIndex, MutableHandleValue rval);
RegExpGetSubstitution(JSContext* cx, HandleObject matchResult, HandleLinearString string,
size_t position, HandleLinearString replacement, size_t firstDollarIndex,
MutableHandleValue rval);
extern MOZ_MUST_USE bool
GetFirstDollarIndex(JSContext* cx, unsigned argc, Value* vp);
extern MOZ_MUST_USE bool
GetFirstDollarIndexRaw(JSContext* cx, HandleString str, int32_t* index);
GetFirstDollarIndexRaw(JSContext* cx, JSString* str, int32_t* index);
extern int32_t
GetFirstDollarIndexRawFlat(JSLinearString* text);

62
js/src/builtin/RegExp.js
View File

@@ -281,28 +281,25 @@ function RegExpReplace(string, replaceValue) {
// Steps 8-16.
if (global) {
// Step 8.a.
var fullUnicode = !!(flags & REGEXP_UNICODE_FLAG);
if (functionalReplace) {
var elemBase = GetElemBaseForLambda(replaceValue);
if (IsObject(elemBase)) {
return RegExpGlobalReplaceOptElemBase(rx, S, lengthS, replaceValue,
fullUnicode, elemBase);
// For large strings check if the replacer function is
// applicable for the elem-base optimization.
if (lengthS > 5000) {
var elemBase = GetElemBaseForLambda(replaceValue);
if (IsObject(elemBase)) {
return RegExpGlobalReplaceOptElemBase(rx, S, lengthS, replaceValue, flags,
elemBase);
}
}
return RegExpGlobalReplaceOptFunc(rx, S, lengthS, replaceValue,
fullUnicode);
return RegExpGlobalReplaceOptFunc(rx, S, lengthS, replaceValue, flags);
}
if (firstDollarIndex !== -1) {
return RegExpGlobalReplaceOptSubst(rx, S, lengthS, replaceValue,
fullUnicode, firstDollarIndex);
return RegExpGlobalReplaceOptSubst(rx, S, lengthS, replaceValue, flags,
firstDollarIndex);
}
if (lengthS < 0x7fff) {
return RegExpGlobalReplaceShortOpt(rx, S, lengthS, replaceValue,
fullUnicode);
}
return RegExpGlobalReplaceOpt(rx, S, lengthS, replaceValue,
fullUnicode);
if (lengthS < 0x7fff)
return RegExpGlobalReplaceShortOpt(rx, S, lengthS, replaceValue, flags);
return RegExpGlobalReplaceOpt(rx, S, lengthS, replaceValue, flags);
}
if (functionalReplace)
@@ -433,8 +430,6 @@ function RegExpReplaceSlowPath(rx, S, lengthS, replaceValue,
// steps 14.g-k.
// Calculates functional/substitution replacement from match result.
// Used in the following functions:
// * RegExpGlobalReplaceOptSubst
// * RegExpLocalReplaceOptSubst
// * RegExpReplaceSlowPath
function RegExpGetComplexReplacement(result, matched, S, position,
nCaptures, replaceValue,
@@ -445,12 +440,7 @@ function RegExpGetComplexReplacement(result, matched, S, position,
var capturesLength = 0;
// Step 14.j.i (reordered).
// For `nCaptures` <= 4 case, call `replaceValue` directly, otherwise
// use `std_Function_apply` with all arguments stored in `captures`.
// In latter case, store `matched` as the first element here, to
// avoid unshift later.
if (functionalReplace && nCaptures > 4)
_DefineDataProperty(captures, capturesLength++, matched);
_DefineDataProperty(captures, capturesLength++, matched);
// Step 14.g, 14.i, 14.i.iv.
for (var n = 1; n <= nCaptures; n++) {
@@ -467,17 +457,19 @@ function RegExpGetComplexReplacement(result, matched, S, position,
// Step 14.j.
if (functionalReplace) {
// For `nCaptures` <= 4 case, call `replaceValue` directly, otherwise
// use `std_Function_apply` with all arguments stored in `captures`.
switch (nCaptures) {
case 0:
return ToString(replaceValue(matched, position, S));
case 1:
return ToString(replaceValue(matched, SPREAD(captures, 1), position, S));
return ToString(replaceValue(SPREAD(captures, 1), position, S));
case 1:
return ToString(replaceValue(SPREAD(captures, 2), position, S));
case 2:
return ToString(replaceValue(matched, SPREAD(captures, 2), position, S));
return ToString(replaceValue(SPREAD(captures, 3), position, S));
case 3:
return ToString(replaceValue(matched, SPREAD(captures, 3), position, S));
return ToString(replaceValue(SPREAD(captures, 4), position, S));
case 4:
return ToString(replaceValue(matched, SPREAD(captures, 4), position, S));
return ToString(replaceValue(SPREAD(captures, 5), position, S));
default:
// Steps 14.j.ii-v.
_DefineDataProperty(captures, capturesLength++, position);
@@ -487,8 +479,7 @@ function RegExpGetComplexReplacement(result, matched, S, position,
}
// Steps 14.k.i.
return RegExpGetSubstitution(matched, S, position, captures, replaceValue,
firstDollarIndex);
return RegExpGetSubstitution(captures, S, position, replaceValue, firstDollarIndex);
}
// ES 2017 draft rev 03bfda119d060aca4099d2b77cf43f6d4f11cfa2 21.2.5.8
@@ -539,8 +530,11 @@ function RegExpGetFunctionalReplacement(result, S, position, replaceValue) {
// * global flag is true
// * S is a short string (lengthS < 0x7fff)
// * replaceValue is a string without "$"
function RegExpGlobalReplaceShortOpt(rx, S, lengthS, replaceValue, fullUnicode)
function RegExpGlobalReplaceShortOpt(rx, S, lengthS, replaceValue, flags)
{
// Step 8.a.
var fullUnicode = !!(flags & REGEXP_UNICODE_FLAG);
// Step 8.b.
var lastIndex = 0;
rx.lastIndex = 0;

16
js/src/builtin/RegExpGlobalReplaceOpt.h.js
View File

@@ -18,7 +18,7 @@
// steps 8.b-16.
// Optimized path for @@replace with the following conditions:
// * global flag is true
function FUNC_NAME(rx, S, lengthS, replaceValue, fullUnicode
function FUNC_NAME(rx, S, lengthS, replaceValue, flags
#ifdef SUBSTITUTION
, firstDollarIndex
#endif
@@ -27,6 +27,9 @@ function FUNC_NAME(rx, S, lengthS, replaceValue, fullUnicode
#endif
)
{
// Step 8.a.
var fullUnicode = !!(flags & REGEXP_UNICODE_FLAG);
// Step 8.b.
var lastIndex = 0;
rx.lastIndex = 0;
@@ -35,7 +38,7 @@ function FUNC_NAME(rx, S, lengthS, replaceValue, fullUnicode
// Save the original source and flags, so we can check if the replacer
// function recompiled the regexp.
var originalSource = UnsafeGetStringFromReservedSlot(rx, REGEXP_SOURCE_SLOT);
var originalFlags = UnsafeGetInt32FromReservedSlot(rx, REGEXP_FLAGS_SLOT);
var originalFlags = flags;
#endif
// Step 12 (reordered).
@@ -53,11 +56,8 @@ function FUNC_NAME(rx, S, lengthS, replaceValue, fullUnicode
if (result === null)
break;
#if defined(SUBSTITUTION)
// Steps 14.a-b.
// Steps 14.a-b (skipped).
assert(result.length >= 1, "RegExpMatcher doesn't return an empty array");
var nCaptures = result.length - 1;
#endif
// Step 14.c.
var matched = result[0];
@@ -74,9 +74,7 @@ function FUNC_NAME(rx, S, lengthS, replaceValue, fullUnicode
#if defined(FUNCTIONAL)
replacement = RegExpGetFunctionalReplacement(result, S, position, replaceValue);
#elif defined(SUBSTITUTION)
replacement = RegExpGetComplexReplacement(result, matched, S, position,
nCaptures, replaceValue,
false, firstDollarIndex);
replacement = RegExpGetSubstitution(result, S, position, replaceValue, firstDollarIndex);
#elif defined(ELEMBASE)
if (IsObject(elemBase)) {
var prop = GetStringDataProperty(elemBase, matched);

19
js/src/builtin/RegExpLocalReplaceOpt.h.js
View File

@@ -76,15 +76,12 @@ function FUNC_NAME(rx, S, lengthS, replaceValue
}
#endif
// Steps 11.c, 12-13, 14.a-b (skipped).
#if defined(SUBSTITUTION)
// Steps 14.a-b.
assert(result.length >= 1, "RegExpMatcher doesn't return an empty array");
var nCaptures = result.length - 1;
#endif
// Steps 11.c, 12-13.
#if !defined(SHORT_STRING)
// Steps 14.a-b.
assert(result.length >= 1, "RegExpMatcher doesn't return an empty array");
// Step 14.c.
var matched = result[0];
@@ -95,10 +92,10 @@ function FUNC_NAME(rx, S, lengthS, replaceValue
var position = result.index;
// Step 14.l.iii (reordered)
// To set rx.lastIndex before RegExpGetComplexReplacement.
// To set rx.lastIndex before RegExpGetFunctionalReplacement.
var nextSourcePosition = position + matchLength;
#else
// Steps 14.c-d (skipped).
// Steps 14.a-d (skipped).
// Step 14.e-f.
var position = result & 0x7fff;
@@ -116,9 +113,7 @@ function FUNC_NAME(rx, S, lengthS, replaceValue
#if defined(FUNCTIONAL)
replacement = RegExpGetFunctionalReplacement(result, S, position, replaceValue);
#elif defined(SUBSTITUTION)
replacement = RegExpGetComplexReplacement(result, matched, S, position,
nCaptures, replaceValue,
false, firstDollarIndex);
replacement = RegExpGetSubstitution(result, S, position, replaceValue, firstDollarIndex);
#else
replacement = replaceValue;
#endif

91
js/src/frontend/Parser.cpp
View File

@@ -1038,8 +1038,8 @@ Parser<ParseHandler, CharT>::parse()
}
/*
* Strict mode forbids introducing new definitions for 'eval', 'arguments', or
* for any strict mode reserved word.
* Strict mode forbids introducing new definitions for 'eval', 'arguments',
* 'let', 'static', 'yield', or for any strict mode reserved word.
*/
bool
ParserBase::isValidStrictBinding(PropertyName* name)
@@ -2285,7 +2285,7 @@ Parser<FullParseHandler, char16_t>::moduleBody(ModuleSharedContext* modulesc)
return null();
AutoAwaitIsKeyword<Parser> awaitIsKeyword(this, AwaitIsModuleKeyword);
ParseNode* pn = statementList(YieldIsKeyword);
ParseNode* pn = statementList(YieldIsName);
if (!pn)
return null();
@@ -4521,7 +4521,12 @@ Parser<ParseHandler, CharT>::objectBindingPattern(DeclarationKind kind,
if (!tokenStream.getToken(&tt))
return null();
Node inner = bindingIdentifierOrPattern(kind, yieldHandling, tt);
if (!TokenKindIsPossibleIdentifierName(tt)) {
error(JSMSG_NO_VARIABLE_NAME);
return null();
}
Node inner = bindingIdentifier(kind, yieldHandling);
if (!inner)
return null();
@@ -5668,7 +5673,7 @@ Parser<ParseHandler, CharT>::exportFunctionDeclaration(uint32_t begin, uint32_t
MOZ_ASSERT(tokenStream.isCurrentTokenType(TOK_FUNCTION));
Node kid = functionStmt(toStringStart, YieldIsKeyword, NameRequired, asyncKind);
Node kid = functionStmt(toStringStart, YieldIsName, NameRequired, asyncKind);
if (!kid)
return null();
@@ -5694,7 +5699,7 @@ Parser<ParseHandler, CharT>::exportClassDeclaration(uint32_t begin)
MOZ_ASSERT(tokenStream.isCurrentTokenType(TOK_CLASS));
Node kid = classDefinition(YieldIsKeyword, ClassStatement, NameRequired);
Node kid = classDefinition(YieldIsName, ClassStatement, NameRequired);
if (!kid)
return null();
@@ -5749,7 +5754,7 @@ Parser<ParseHandler, CharT>::exportDefaultFunctionDeclaration(uint32_t begin,
MOZ_ASSERT(tokenStream.isCurrentTokenType(TOK_FUNCTION));
Node kid = functionStmt(toStringStart, YieldIsKeyword, AllowDefaultName, asyncKind);
Node kid = functionStmt(toStringStart, YieldIsName, AllowDefaultName, asyncKind);
if (!kid)
return null();
@@ -5772,7 +5777,7 @@ Parser<ParseHandler, CharT>::exportDefaultClassDeclaration(uint32_t begin)
MOZ_ASSERT(tokenStream.isCurrentTokenType(TOK_CLASS));
Node kid = classDefinition(YieldIsKeyword, ClassStatement, AllowDefaultName);
Node kid = classDefinition(YieldIsName, ClassStatement, AllowDefaultName);
if (!kid)
return null();
@@ -5800,7 +5805,7 @@ Parser<ParseHandler, CharT>::exportDefaultAssignExpr(uint32_t begin)
if (!noteDeclaredName(name, DeclarationKind::Const, pos()))
return null();
Node kid = assignExpr(InAllowed, YieldIsKeyword, TripledotProhibited);
Node kid = assignExpr(InAllowed, YieldIsName, TripledotProhibited);
if (!kid)
return null();
if (!matchOrInsertSemicolonAfterExpression())
@@ -6167,7 +6172,7 @@ Parser<ParseHandler, CharT>::forHeadStart(YieldHandling yieldHandling,
if (!tokenStream.peekToken(&next))
return false;
parsingLexicalDeclaration = nextTokenContinuesLetDeclaration(next, yieldHandling);
parsingLexicalDeclaration = nextTokenContinuesLetDeclaration(next);
if (!parsingLexicalDeclaration) {
tokenStream.ungetToken();
letIsIdentifier = true;
@@ -7467,8 +7472,7 @@ Parser<ParseHandler, CharT>::classDefinition(YieldHandling yieldHandling,
template <class ParseHandler, typename CharT>
bool
Parser<ParseHandler, CharT>::nextTokenContinuesLetDeclaration(TokenKind next,
YieldHandling yieldHandling)
Parser<ParseHandler, CharT>::nextTokenContinuesLetDeclaration(TokenKind next)
{
MOZ_ASSERT(tokenStream.isCurrentTokenType(TOK_LET));
@@ -7478,42 +7482,23 @@ Parser<ParseHandler, CharT>::nextTokenContinuesLetDeclaration(TokenKind next,
MOZ_ASSERT(next == verify);
#endif
// Destructuring is (for once) the easy case.
// Destructuring continues a let declaration.
if (next == TOK_LB || next == TOK_LC)
return true;
// If we have the name "yield", the grammar parameter exactly states
// whether this is okay. (This wasn't true for SpiderMonkey's ancient
// legacy generator syntax, but that's dead now.) If YieldIsName,
// declaration-parsing code will (if necessary) enforce a strict mode
// restriction on defining "yield". If YieldIsKeyword, consider this the
// end of the declaration, in case ASI induces a semicolon that makes the
// "yield" valid.
if (next == TOK_YIELD)
return yieldHandling == YieldIsName;
// Somewhat similar logic applies for "await", except that it's not tracked
// with an AwaitHandling argument.
if (next == TOK_AWAIT)
return !awaitIsKeyword();
// A "let" edge case deserves special comment. Consider this:
//
// let // not an ASI opportunity
// let;
//
// Static semantics in §13.3.1.1 turn a LexicalDeclaration that binds
// "let" into an early error. Does this retroactively permit ASI so
// that we should parse this as two ExpressionStatements? No. ASI
// resolves during parsing. Static semantics only apply to the full
// parse tree with ASI applied. No backsies!
// Otherwise a let declaration must have a name.
if (TokenKindIsPossibleIdentifier(next)) {
// A "let" edge case deserves special comment. Consider this:
//
// let // not an ASI opportunity
// let;
//
// Static semantics in §13.3.1.1 turn a LexicalDeclaration that binds
// "let" into an early error. Does this retroactively permit ASI so
// that we should parse this as two ExpressionStatements? No. ASI
// resolves during parsing. Static semantics only apply to the full
// parse tree with ASI applied. No backsies!
return true;
}
// Otherwise not a let declaration.
return false;
return TokenKindIsPossibleIdentifier(next);
}
template <class ParseHandler, typename CharT>
@@ -7816,7 +7801,7 @@ Parser<ParseHandler, CharT>::statementListItem(YieldHandling yieldHandling,
if (!tokenStream.peekToken(&next))
return null();
if (tt == TOK_LET && nextTokenContinuesLetDeclaration(next, yieldHandling))
if (tt == TOK_LET && nextTokenContinuesLetDeclaration(next))
return lexicalDeclaration(yieldHandling, DeclarationKind::Let);
if (tt == TOK_ASYNC) {
@@ -9501,7 +9486,8 @@ Parser<SyntaxParseHandler, char16_t>::newRegExp()
template <class ParseHandler, typename CharT>
void
Parser<ParseHandler, CharT>::checkDestructuringAssignmentTarget(Node expr, TokenPos exprPos,
PossibleError* possibleError)
PossibleError* possibleError,
TargetBehavior behavior)
{
// Return early if a pending destructuring error is already present.
if (possibleError->hasPendingDestructuringError())
@@ -9531,6 +9517,15 @@ Parser<ParseHandler, CharT>::checkDestructuringAssignmentTarget(Node expr, Token
}
}
if (behavior == TargetBehavior::ForbidAssignmentPattern) {
if (handler.isUnparenthesizedDestructuringPattern(expr) ||
handler.isParenthesizedDestructuringPattern(expr))
{
possibleError->setPendingDestructuringErrorAt(exprPos, JSMSG_BAD_DESTRUCT_TARGET);
return;
}
}
// The expression must be either a simple assignment target, i.e. a name
// or a property accessor, or a nested destructuring pattern.
if (!handler.isUnparenthesizedDestructuringPattern(expr) &&
@@ -9932,8 +9927,10 @@ Parser<ParseHandler, CharT>::objectLiteral(YieldHandling yieldHandling,
possibleError);
if (!inner)
return null();
if (possibleError)
checkDestructuringAssignmentTarget(inner, innerPos, possibleError);
if (possibleError) {
checkDestructuringAssignmentTarget(inner, innerPos, possibleError,
TargetBehavior::ForbidAssignmentPattern);
}
if (!handler.addSpreadProperty(literal, begin, inner))
return null();
} else {

9
js/src/frontend/Parser.h
View File

@@ -660,7 +660,7 @@ class Parser final : public ParserBase, private JS::AutoGCRooter
// While on a |let| TOK_NAME token, examine |next|. Indicate whether
// |next|, the next token already gotten with modifier TokenStream::None,
// continues a LexicalDeclaration.
bool nextTokenContinuesLetDeclaration(TokenKind next, YieldHandling yieldHandling);
bool nextTokenContinuesLetDeclaration(TokenKind next);
Node lexicalDeclaration(YieldHandling yieldHandling, DeclarationKind kind);
@@ -934,8 +934,13 @@ class Parser final : public ParserBase, private JS::AutoGCRooter
Node objectBindingPattern(DeclarationKind kind, YieldHandling yieldHandling);
Node arrayBindingPattern(DeclarationKind kind, YieldHandling yieldHandling);
enum class TargetBehavior {
PermitAssignmentPattern,
ForbidAssignmentPattern
};
void checkDestructuringAssignmentTarget(Node expr, TokenPos exprPos,
PossibleError* possibleError);
PossibleError* possibleError,
TargetBehavior behavior = TargetBehavior::PermitAssignmentPattern);
void checkDestructuringAssignmentElement(Node expr, TokenPos exprPos,
PossibleError* possibleError);

2
js/src/jit/CodeGenerator.cpp
View File

@@ -2417,7 +2417,7 @@ FindFirstDollarIndex(MacroAssembler& masm, Register str, Register len, Register
masm.bind(&done);
}
typedef bool (*GetFirstDollarIndexRawFn)(JSContext*, HandleString, int32_t*);
typedef bool (*GetFirstDollarIndexRawFn)(JSContext*, JSString*, int32_t*);
static const VMFunction GetFirstDollarIndexRawInfo =
FunctionInfo<GetFirstDollarIndexRawFn>(GetFirstDollarIndexRaw, "GetFirstDollarIndexRaw");

10
js/src/tests/js1_8_5/reflect-parse/object-rest.js
View File

@@ -13,18 +13,14 @@ function assertDestrBinding(src, pattern) {
}
function test() {
// Target expression must be a simple assignment target or a nested pattern
// in object assignment patterns.
// Target expression must be a simple assignment target in object assignment patterns.
assertDestrAssign("{...x}", objPatt([spread(ident("x"))]));
assertDestrAssign("{...(x)}", objPatt([spread(ident("x"))]));
assertDestrAssign("{...obj.p}", objPatt([spread(dotExpr(ident("obj"), ident("p")))]));
assertDestrAssign("{...{}}", objPatt([spread(objPatt([]))]));
assertDestrAssign("{...[]}", objPatt([spread(arrPatt([]))]));
assertDestrAssign("{...(obj.p)}", objPatt([spread(dotExpr(ident("obj"), ident("p")))]));
// Object binding patterns only allow binding identifiers or nested patterns.
// Object binding patterns only allow binding identifiers.
assertDestrBinding("{...x}", objPatt([spread(ident("x"))]));
assertDestrBinding("{...{}}", objPatt([spread(objPatt([]))]));
assertDestrBinding("{...[]}", objPatt([spread(arrPatt([]))]));
// The rest-property can be preceded by other properties.
for (var assertDestr of [assertDestrAssign, assertDestrBinding]) {

212
js/src/tests/jstests.list
View File

@@ -532,3 +532,215 @@ skip script test262/language/statements/with/decl-async-gen.js
####################################################
# Tests disabled due to invalid test expectations #
####################################################
# https://github.com/tc39/test262/pull/1072
skip script test262/language/statements/for-await-of/async-gen-dstr-let-async-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/for-await-of/async-func-dstr-var-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/for-await-of/async-func-dstr-var-async-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/for-await-of/async-func-dstr-var-async-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/for-await-of/async-func-dstr-let-async-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/for-await-of/async-func-dstr-const-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/for-await-of/async-func-dstr-const-async-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/for-await-of/async-gen-dstr-let-async-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/for-await-of/async-func-dstr-let-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/for-await-of/async-gen-dstr-var-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/for-await-of/async-gen-dstr-var-async-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/for-await-of/async-gen-dstr-const-async-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/for-await-of/async-func-dstr-let-async-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/for-await-of/async-gen-dstr-var-async-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/for-await-of/async-func-dstr-const-async-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/for-await-of/async-func-dstr-let-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/for-await-of/async-func-dstr-let-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/for-await-of/async-gen-dstr-let-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/for-await-of/async-gen-dstr-const-async-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/for-await-of/async-func-dstr-var-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/for-await-of/async-gen-dstr-let-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/for-await-of/async-gen-dstr-const-async-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/for-await-of/async-gen-dstr-var-async-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/for-await-of/async-gen-dstr-var-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/for-await-of/async-gen-dstr-const-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/for-await-of/async-gen-dstr-const-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/for-await-of/async-gen-dstr-let-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/for-await-of/async-func-dstr-var-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/for-await-of/async-gen-dstr-const-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/for-await-of/async-func-dstr-const-async-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/for-await-of/async-func-dstr-const-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/for-await-of/async-func-dstr-let-async-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/for-await-of/async-func-dstr-const-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/for-await-of/async-func-dstr-var-async-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/for-await-of/async-gen-dstr-let-async-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/for-await-of/async-gen-dstr-var-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/generators/dstr-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/generators/dstr-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/generators/dstr-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/generators/dstr-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/generators/dstr-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/generators/dstr-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/let/dstr-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/let/dstr-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/let/dstr-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/const/dstr-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/const/dstr-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/const/dstr-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/async-generator/dstr-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/async-generator/dstr-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/async-generator/dstr-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/async-generator/dstr-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/async-generator/dstr-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/async-generator/dstr-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/variable/dstr-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/variable/dstr-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/variable/dstr-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/try/dstr-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/try/dstr-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/try/dstr-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/function/dstr-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/function/dstr-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/function/dstr-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/function/dstr-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/function/dstr-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/function/dstr-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/for/dstr-let-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/for/dstr-let-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/for/dstr-const-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/for/dstr-var-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/for/dstr-var-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/for/dstr-const-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/for/dstr-const-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/for/dstr-let-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/for/dstr-var-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/for-of/dstr-let-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/for-of/dstr-obj-rest-nested-obj-nested-rest.js
skip script test262/language/statements/for-of/dstr-let-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/for-of/dstr-const-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/for-of/dstr-var-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/for-of/dstr-var-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/for-of/dstr-obj-rest-nested-obj.js
skip script test262/language/statements/for-of/dstr-obj-rest-obj-own-property.js
skip script test262/language/statements/for-of/dstr-const-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/for-of/dstr-const-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/for-of/dstr-let-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/for-of/dstr-var-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/class/dstr-async-gen-meth-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/class/dstr-gen-meth-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/class/dstr-gen-meth-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/class/dstr-async-gen-meth-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/class/dstr-meth-static-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/class/dstr-gen-meth-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/class/dstr-meth-static-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/class/dstr-meth-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/class/dstr-meth-static-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/class/dstr-meth-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/class/dstr-async-gen-meth-static-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/class/dstr-meth-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/class/dstr-async-gen-meth-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/class/dstr-meth-static-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/class/dstr-async-gen-meth-static-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/class/dstr-gen-meth-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/class/dstr-meth-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/class/dstr-gen-meth-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/class/dstr-meth-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/class/dstr-async-gen-meth-static-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/class/dstr-gen-meth-static-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/class/dstr-gen-meth-static-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/class/dstr-meth-static-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/class/dstr-gen-meth-static-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/class/dstr-async-gen-meth-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/class/dstr-async-gen-meth-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/class/dstr-async-gen-meth-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/class/dstr-gen-meth-static-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/class/dstr-meth-obj-ptrn-rest-nested-obj.js
skip script test262/language/statements/class/dstr-async-gen-meth-static-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/class/dstr-async-gen-meth-static-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/class/dstr-gen-meth-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/class/dstr-meth-static-obj-ptrn-rest-obj-own-property.js
skip script test262/language/statements/class/dstr-gen-meth-static-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/class/dstr-async-gen-meth-static-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/statements/class/dstr-gen-meth-static-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/generators/dstr-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/generators/dstr-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/generators/dstr-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/generators/dstr-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/generators/dstr-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/generators/dstr-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/async-generator/dstr-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/async-generator/dstr-named-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/async-generator/dstr-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/async-generator/dstr-named-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/async-generator/dstr-named-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/async-generator/dstr-named-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/async-generator/dstr-named-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/async-generator/dstr-named-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/async-generator/dstr-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/async-generator/dstr-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/async-generator/dstr-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/async-generator/dstr-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/assignment/dstr-obj-rest-nested-obj-nested-rest.js
skip script test262/language/expressions/assignment/dstr-obj-rest-nested-obj.js
skip script test262/language/expressions/assignment/dstr-obj-rest-obj-own-property.js
skip script test262/language/expressions/function/dstr-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/function/dstr-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/function/dstr-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/function/dstr-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/function/dstr-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/function/dstr-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/object/dstr-async-gen-meth-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/object/dstr-gen-meth-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/object/dstr-gen-meth-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/object/dstr-async-gen-meth-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/object/dstr-gen-meth-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/object/dstr-meth-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/object/dstr-meth-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/object/dstr-meth-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/object/dstr-async-gen-meth-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/object/dstr-gen-meth-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/object/dstr-meth-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/object/dstr-gen-meth-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/object/dstr-meth-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/object/dstr-async-gen-meth-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/object/dstr-async-gen-meth-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/object/dstr-async-gen-meth-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/object/dstr-meth-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/object/dstr-gen-meth-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/arrow-function/dstr-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/arrow-function/dstr-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/arrow-function/dstr-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/arrow-function/dstr-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/arrow-function/dstr-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/arrow-function/dstr-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/class/dstr-async-gen-meth-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/class/dstr-gen-meth-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/class/dstr-gen-meth-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/class/dstr-async-gen-meth-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/class/dstr-meth-static-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/class/dstr-gen-meth-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/class/dstr-meth-static-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/class/dstr-meth-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/class/dstr-meth-static-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/class/dstr-meth-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/class/dstr-async-gen-meth-static-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/class/dstr-meth-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/class/dstr-async-gen-meth-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/class/dstr-meth-static-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/class/dstr-async-gen-meth-static-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/class/dstr-gen-meth-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/class/dstr-meth-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/class/dstr-gen-meth-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/class/dstr-meth-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/class/dstr-async-gen-meth-static-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/class/dstr-gen-meth-static-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/class/dstr-gen-meth-static-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/class/dstr-meth-static-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/class/dstr-gen-meth-static-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/class/dstr-async-gen-meth-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/class/dstr-async-gen-meth-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/class/dstr-async-gen-meth-dflt-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/class/dstr-gen-meth-static-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/class/dstr-meth-obj-ptrn-rest-nested-obj.js
skip script test262/language/expressions/class/dstr-async-gen-meth-static-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/class/dstr-async-gen-meth-static-dflt-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/class/dstr-gen-meth-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/class/dstr-meth-static-obj-ptrn-rest-obj-own-property.js
skip script test262/language/expressions/class/dstr-gen-meth-static-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/class/dstr-async-gen-meth-static-dflt-obj-ptrn-rest-obj-nested-rest.js
skip script test262/language/expressions/class/dstr-gen-meth-static-dflt-obj-ptrn-rest-nested-obj.js

18
js/src/tests/test262/local/let-newline-await-in-async-function.js
View File

@@ -1,18 +0,0 @@
// Copyright (C) 2017 Mozilla Corporation. All rights reserved.
// This code is governed by the BSD license found in the LICENSE file.
/*---
author: Jeff Walden <jwalden+code@mit.edu>
esid: sec-let-and-const-declarations
description: >
|await| is excluded from LexicalDeclaration by grammar parameter, in
AsyncFunction. Therefore |let| followed by |await| inside AsyncFunction is
an ASI opportunity, and this code must parse without error.
---*/
async function f() {
let
await 0;
}
reportCompare(true, f instanceof Function);

20
js/src/tests/test262/local/let-newline-await-in-normal-function.js
View File

@@ -1,20 +0,0 @@
// |reftest| error:SyntaxError
// Copyright (C) 2017 Mozilla Corporation. All rights reserved.
// This code is governed by the BSD license found in the LICENSE file.
/*---
author: Jeff Walden <jwalden+code@mit.edu>
esid: sec-let-and-const-declarations
description: >
Outside AsyncFunction, |await| is a perfectly cromulent LexicalDeclaration
variable name. Therefore ASI doesn't apply, and so the |0| where a |=| was
expected is a syntax error.
negative:
phase: early
type: SyntaxError
---*/
function f() {
let
await 0;
}

37
js/src/vm/SelfHosting.cpp
View File

@@ -1530,39 +1530,26 @@ static bool
intrinsic_RegExpGetSubstitution(JSContext* cx, unsigned argc, Value* vp)
{
CallArgs args = CallArgsFromVp(argc, vp);
MOZ_ASSERT(args.length() == 5);
MOZ_ASSERT(args.length() == 6);
RootedObject matchResult(cx, &args[0].toObject());
RootedString matched(cx, args[0].toString());
RootedString string(cx, args[1].toString());
RootedLinearString string(cx, args[1].toString()->ensureLinear(cx));
if (!string)
return false;
int32_t position = int32_t(args[2].toNumber());
MOZ_ASSERT(position >= 0);
RootedObject captures(cx, &args[3].toObject());
#ifdef DEBUG
bool isArray = false;
MOZ_ALWAYS_TRUE(IsArray(cx, captures, &isArray));
MOZ_ASSERT(isArray);
#endif
RootedLinearString replacement(cx, args[3].toString()->ensureLinear(cx));
if (!replacement)
return false;
RootedString replacement(cx, args[4].toString());
int32_t firstDollarIndex = int32_t(args[5].toNumber());
int32_t firstDollarIndex = int32_t(args[4].toNumber());
MOZ_ASSERT(firstDollarIndex >= 0);
RootedLinearString matchedLinear(cx, matched->ensureLinear(cx));
if (!matchedLinear)
return false;
RootedLinearString stringLinear(cx, string->ensureLinear(cx));
if (!stringLinear)
return false;
RootedLinearString replacementLinear(cx, replacement->ensureLinear(cx));
if (!replacementLinear)
return false;
return RegExpGetSubstitution(cx, matchedLinear, stringLinear, size_t(position), captures,
replacementLinear, size_t(firstDollarIndex), args.rval());
return RegExpGetSubstitution(cx, matchResult, string, size_t(position), replacement,
size_t(firstDollarIndex), args.rval());
}
static bool
@@ -2549,7 +2536,7 @@ static const JSFunctionSpec intrinsic_functions[] = {
RegExpPrototypeOptimizable),
JS_INLINABLE_FN("RegExpInstanceOptimizable", RegExpInstanceOptimizable, 1,0,
RegExpInstanceOptimizable),
JS_FN("RegExpGetSubstitution", intrinsic_RegExpGetSubstitution, 6,0),
JS_FN("RegExpGetSubstitution", intrinsic_RegExpGetSubstitution, 5,0),
JS_FN("GetElemBaseForLambda", intrinsic_GetElemBaseForLambda, 1,0),
JS_FN("GetStringDataProperty", intrinsic_GetStringDataProperty, 2,0),
JS_INLINABLE_FN("GetFirstDollarIndex", GetFirstDollarIndex, 1,0,

1
layout/inspector/moz.build
View File

@@ -50,5 +50,4 @@ LOCAL_INCLUDES += [
'../style',
'/dom/base',
'/dom/xbl',
'/modules/brotli/dec',
]

10
layout/inspector/nsFontFace.cpp
View File

@@ -9,7 +9,7 @@
#include "gfxUserFontSet.h"
#include "nsFontFaceLoader.h"
#include "mozilla/gfx/2D.h"
#include "decode.h"
#include "brotli/decode.h"
#include "zlib.h"
#include "mozilla/dom/FontFaceSet.h"
@@ -208,10 +208,10 @@ nsFontFace::GetMetadata(nsAString & aMetadata)
case gfxUserFontData::kBrotliCompression:
{
size_t decodedSize = userFontData->mMetaOrigLen;
if (BrotliDecompressBuffer(userFontData->mMetadata.Length(),
userFontData->mMetadata.Elements(),
&decodedSize,
(uint8_t*)str.BeginWriting()) == 1 &&
if (BrotliDecoderDecompress(userFontData->mMetadata.Length(),
userFontData->mMetadata.Elements(),
&decodedSize,
(uint8_t*)str.BeginWriting()) == 1 &&
decodedSize == userFontData->mMetaOrigLen) {
AppendUTF8toUTF16(str, aMetadata);
}

2
modules/brotli/README.mozilla
View File

@@ -14,4 +14,4 @@ The in-tree copy is updated by running
sh update.sh
from within the modules/brotli directory.
Current version: [commit 29d31d5921b0a2b323ac24e7f7d0cdc9a3c0dd08].
Current version: [commit 46c1a881b41bb638c76247558aa04b1591af3aa7].

39
modules/brotli/clang-cl-exceptions.patch
View File

@@ -1,39 +0,0 @@
diff --git a/tools/bro.cc b/tools/bro.cc
index b254f0ffdb08..30d9858a95b3 100644
--- a/tools/bro.cc
+++ b/tools/bro.cc
@@ -282,29 +282,34 @@ int main(int argc, char** argv) {
FILE* fin = OpenInputFile(input_path);
FILE* fout = OpenOutputFile(output_path, force);
if (decompress) {
Decompresss(fin, fout);
} else {
brotli::BrotliParams params;
params.lgwin = lgwin;
params.quality = quality;
+/* clang-cl doesn't like exceptions */
+#if !defined(_MSC_VER) || !defined(__clang__)
try {
+#endif
brotli::BrotliFileIn in(fin, 1 << 16);
brotli::BrotliFileOut out(fout);
if (!BrotliCompress(params, &in, &out)) {
fprintf(stderr, "compression failed\n");
unlink(output_path);
exit(1);
}
+#if !defined(_MSC_VER) || !defined(__clang__)
} catch (std::bad_alloc&) {
fprintf(stderr, "not enough memory\n");
unlink(output_path);
exit(1);
}
+#endif
}
if (fclose(fin) != 0) {
perror("fclose");
exit(1);
}
if (fclose(fout) != 0) {
perror("fclose");
exit(1);

55
modules/brotli/common/constants.h Normal file
View File

@@ -0,0 +1,55 @@
/* Copyright 2016 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
#ifndef BROTLI_COMMON_CONSTANTS_H_
#define BROTLI_COMMON_CONSTANTS_H_
/* Specification: 7.3. Encoding of the context map */
#define BROTLI_CONTEXT_MAP_MAX_RLE 16
/* Specification: 2. Compressed representation overview */
#define BROTLI_MAX_NUMBER_OF_BLOCK_TYPES 256
/* Specification: 3.3. Alphabet sizes: insert-and-copy length */
#define BROTLI_NUM_LITERAL_SYMBOLS 256
#define BROTLI_NUM_COMMAND_SYMBOLS 704
#define BROTLI_NUM_BLOCK_LEN_SYMBOLS 26
#define BROTLI_MAX_CONTEXT_MAP_SYMBOLS (BROTLI_MAX_NUMBER_OF_BLOCK_TYPES + \
BROTLI_CONTEXT_MAP_MAX_RLE)
#define BROTLI_MAX_BLOCK_TYPE_SYMBOLS (BROTLI_MAX_NUMBER_OF_BLOCK_TYPES + 2)
/* Specification: 3.5. Complex prefix codes */
#define BROTLI_REPEAT_PREVIOUS_CODE_LENGTH 16
#define BROTLI_REPEAT_ZERO_CODE_LENGTH 17
#define BROTLI_CODE_LENGTH_CODES (BROTLI_REPEAT_ZERO_CODE_LENGTH + 1)
/* "code length of 8 is repeated" */
#define BROTLI_INITIAL_REPEATED_CODE_LENGTH 8
/* Specification: 4. Encoding of distances */
#define BROTLI_NUM_DISTANCE_SHORT_CODES 16
#define BROTLI_MAX_NPOSTFIX 3
#define BROTLI_MAX_NDIRECT 120
#define BROTLI_MAX_DISTANCE_BITS 24U
/* BROTLI_NUM_DISTANCE_SYMBOLS == 520 */
#define BROTLI_NUM_DISTANCE_SYMBOLS (BROTLI_NUM_DISTANCE_SHORT_CODES + \
BROTLI_MAX_NDIRECT + \
(BROTLI_MAX_DISTANCE_BITS << \
(BROTLI_MAX_NPOSTFIX + 1)))
/* 7.1. Context modes and context ID lookup for literals */
/* "context IDs for literals are in the range of 0..63" */
#define BROTLI_LITERAL_CONTEXT_BITS 6
/* 7.2. Context ID for distances */
#define BROTLI_DISTANCE_CONTEXT_BITS 2
/* 9.1. Format of the Stream Header */
/* Number of slack bytes for window size. Don't confuse
with BROTLI_NUM_DISTANCE_SHORT_CODES. */
#define BROTLI_WINDOW_GAP 16
#define BROTLI_MAX_BACKWARD_LIMIT(W) (((size_t)1 << (W)) - BROTLI_WINDOW_GAP)
#endif /* BROTLI_COMMON_CONSTANTS_H_ */

5886
modules/brotli/common/dictionary.c Normal file
View File

File diff suppressed because it is too large Load Diff

50
modules/brotli/common/dictionary.h Normal file
View File

@@ -0,0 +1,50 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Collection of static dictionary words. */
#ifndef BROTLI_COMMON_DICTIONARY_H_
#define BROTLI_COMMON_DICTIONARY_H_
#include <brotli/port.h>
#include <brotli/types.h>
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
typedef struct BrotliDictionary {
/**
* Number of bits to encode index of dictionary word in a bucket.
*
* Specification: Appendix A. Static Dictionary Data
*
* Words in a dictionary are bucketed by length.
* @c 0 means that there are no words of a given length.
* Dictionary consists of words with length of [4..24] bytes.
* Values at [0..3] and [25..31] indices should not be addressed.
*/
uint8_t size_bits_by_length[32];
/* assert(offset[i + 1] == offset[i] + (bits[i] ? (i << bits[i]) : 0)) */
uint32_t offsets_by_length[32];
/* Data array is not bound, and should obey to size_bits_by_length values.
Specified size matches default (RFC 7932) dictionary. */
/* assert(sizeof(data) == offsets_by_length[31]) */
uint8_t data[122784];
} BrotliDictionary;
BROTLI_COMMON_API extern const BrotliDictionary* BrotliGetDictionary(void);
#define BROTLI_MIN_DICTIONARY_WORD_LENGTH 4
#define BROTLI_MAX_DICTIONARY_WORD_LENGTH 24
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_COMMON_DICTIONARY_H_ */

19
modules/brotli/common/version.h Executable file
View File

@@ -0,0 +1,19 @@
/* Copyright 2016 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Version definition. */
#ifndef BROTLI_COMMON_VERSION_H_
#define BROTLI_COMMON_VERSION_H_
/* This macro should only be used when library is compiled together with client.
If library is dynamically linked, use BrotliDecoderVersion and
BrotliEncoderVersion methods. */
/* Semantic version, calculated as (MAJOR << 24) | (MINOR << 12) | PATCH */
#define BROTLI_VERSION 0x0006000
#endif /* BROTLI_COMMON_VERSION_H_ */

12
modules/brotli/dec/Makefile
View File

@@ -1,12 +0,0 @@
#brotli/dec
include ../shared.mk
CFLAGS += -Wall
OBJS = bit_reader.o decode.o dictionary.o huffman.o state.o
all : $(OBJS)
clean :
rm -f $(OBJS)

12
modules/brotli/dec/bit_reader.c
View File

@@ -8,8 +8,8 @@
#include "./bit_reader.h"
#include <brotli/types.h>
#include "./port.h"
#include "./types.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -20,27 +20,27 @@ void BrotliInitBitReader(BrotliBitReader* const br) {
br->bit_pos_ = sizeof(br->val_) << 3;
}
int BrotliWarmupBitReader(BrotliBitReader* const br) {
BROTLI_BOOL BrotliWarmupBitReader(BrotliBitReader* const br) {
size_t aligned_read_mask = (sizeof(br->val_) >> 1) - 1;
/* Fixing alignment after unaligned BrotliFillWindow would result accumulator
overflow. If unalignment is caused by BrotliSafeReadBits, then there is
enough space in accumulator to fix aligment. */
enough space in accumulator to fix alignment. */
if (!BROTLI_ALIGNED_READ) {
aligned_read_mask = 0;
}
if (BrotliGetAvailableBits(br) == 0) {
if (!BrotliPullByte(br)) {
return 0;
return BROTLI_FALSE;
}
}
while ((((size_t)br->next_in) & aligned_read_mask) != 0) {
if (!BrotliPullByte(br)) {
/* If we consumed all the input, we don't care about the alignment. */
return 1;
return BROTLI_TRUE;
}
}
return 1;
return BROTLI_TRUE;
}
#if defined(__cplusplus) || defined(c_plusplus)

73
modules/brotli/dec/bit_reader.h
View File

@@ -11,20 +11,14 @@
#include <string.h> /* memcpy */
#include <brotli/types.h>
#include "./port.h"
#include "./types.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
#if (BROTLI_64_BITS)
#define BROTLI_SHORT_FILL_BIT_WINDOW_READ 4
typedef uint64_t reg_t;
#else
#define BROTLI_SHORT_FILL_BIT_WINDOW_READ 2
typedef uint32_t reg_t;
#endif
#define BROTLI_SHORT_FILL_BIT_WINDOW_READ (sizeof(reg_t) >> 1)
static const uint32_t kBitMask[33] = { 0x0000,
0x00000001, 0x00000003, 0x00000007, 0x0000000F,
@@ -61,14 +55,14 @@ typedef struct {
size_t avail_in;
} BrotliBitReaderState;
/* Initializes the bitreader fields. */
/* Initializes the BrotliBitReader fields. */
BROTLI_INTERNAL void BrotliInitBitReader(BrotliBitReader* const br);
/* Ensures that accumulator is not empty. May consume one byte of input.
Returns 0 if data is required but there is no input available.
For BROTLI_ALIGNED_READ this function also prepares bit reader for aligned
reading. */
BROTLI_INTERNAL int BrotliWarmupBitReader(BrotliBitReader* const br);
BROTLI_INTERNAL BROTLI_BOOL BrotliWarmupBitReader(BrotliBitReader* const br);
static BROTLI_INLINE void BrotliBitReaderSaveState(
BrotliBitReader* const from, BrotliBitReaderState* to) {
@@ -97,11 +91,11 @@ static BROTLI_INLINE size_t BrotliGetRemainingBytes(BrotliBitReader* br) {
return br->avail_in + (BrotliGetAvailableBits(br) >> 3);
}
/* Checks if there is at least num bytes left in the input ringbuffer (excluding
the bits remaining in br->val_). */
static BROTLI_INLINE int BrotliCheckInputAmount(
/* Checks if there is at least |num| bytes left in the input ring-buffer
(excluding the bits remaining in br->val_). */
static BROTLI_INLINE BROTLI_BOOL BrotliCheckInputAmount(
BrotliBitReader* const br, size_t num) {
return br->avail_in >= num;
return TO_BROTLI_BOOL(br->avail_in >= num);
}
static BROTLI_INLINE uint16_t BrotliLoad16LE(const uint8_t* in) {
@@ -163,7 +157,7 @@ static BROTLI_INLINE uint64_t BrotliLoad64LE(const uint8_t* in) {
/* Guarantees that there are at least n_bits + 1 bits in accumulator.
Precondition: accumulator contains at least 1 bit.
n_bits should be in the range [1..24] for regular build. For portable
non-64-bit little endian build only 16 bits are safe to request. */
non-64-bit little-endian build only 16 bits are safe to request. */
static BROTLI_INLINE void BrotliFillBitWindow(
BrotliBitReader* const br, uint32_t n_bits) {
#if (BROTLI_64_BITS)
@@ -213,16 +207,16 @@ static BROTLI_INLINE void BrotliFillBitWindow(
#endif
}
/* Mosltly like BrotliFillBitWindow, but guarantees only 16 bits and reads no
/* Mostly like BrotliFillBitWindow, but guarantees only 16 bits and reads no
more than BROTLI_SHORT_FILL_BIT_WINDOW_READ bytes of input. */
static BROTLI_INLINE void BrotliFillBitWindow16(BrotliBitReader* const br) {
BrotliFillBitWindow(br, 17);
}
/* Pulls one byte of input to accumulator. */
static BROTLI_INLINE int BrotliPullByte(BrotliBitReader* const br) {
static BROTLI_INLINE BROTLI_BOOL BrotliPullByte(BrotliBitReader* const br) {
if (br->avail_in == 0) {
return 0;
return BROTLI_FALSE;
}
br->val_ >>= 8;
#if (BROTLI_64_BITS)
@@ -233,11 +227,11 @@ static BROTLI_INLINE int BrotliPullByte(BrotliBitReader* const br) {
br->bit_pos_ -= 8;
--br->avail_in;
++br->next_in;
return 1;
return BROTLI_TRUE;
}
/* Returns currently available bits.
The number of valid bits could be calclulated by BrotliGetAvailableBits. */
The number of valid bits could be calculated by BrotliGetAvailableBits. */
static BROTLI_INLINE reg_t BrotliGetBitsUnmasked(BrotliBitReader* const br) {
return br->val_ >> br->bit_pos_;
}
@@ -250,7 +244,7 @@ static BROTLI_INLINE uint32_t BrotliGet16BitsUnmasked(
return (uint32_t)BrotliGetBitsUnmasked(br);
}
/* Returns the specified number of bits from br without advancing bit pos. */
/* Returns the specified number of bits from |br| without advancing bit pos. */
static BROTLI_INLINE uint32_t BrotliGetBits(
BrotliBitReader* const br, uint32_t n_bits) {
BrotliFillBitWindow(br, n_bits);
@@ -259,15 +253,15 @@ static BROTLI_INLINE uint32_t BrotliGetBits(
/* Tries to peek the specified amount of bits. Returns 0, if there is not
enough input. */
static BROTLI_INLINE int BrotliSafeGetBits(
static BROTLI_INLINE BROTLI_BOOL BrotliSafeGetBits(
BrotliBitReader* const br, uint32_t n_bits, uint32_t* val) {
while (BrotliGetAvailableBits(br) < n_bits) {
if (!BrotliPullByte(br)) {
return 0;
return BROTLI_FALSE;
}
}
*val = (uint32_t)BrotliGetBitsUnmasked(br) & BitMask(n_bits);
return 1;
return BROTLI_TRUE;
}
/* Advances the bit pos by n_bits. */
@@ -289,7 +283,7 @@ static BROTLI_INLINE void BrotliBitReaderUnload(BrotliBitReader* br) {
br->bit_pos_ += unused_bits;
}
/* Reads the specified number of bits from br and advances the bit pos.
/* Reads the specified number of bits from |br| and advances the bit pos.
Precondition: accumulator MUST contain at least n_bits. */
static BROTLI_INLINE void BrotliTakeBits(
BrotliBitReader* const br, uint32_t n_bits, uint32_t* val) {
@@ -299,7 +293,7 @@ static BROTLI_INLINE void BrotliTakeBits(
BrotliDropBits(br, n_bits);
}
/* Reads the specified number of bits from br and advances the bit pos.
/* Reads the specified number of bits from |br| and advances the bit pos.
Assumes that there is enough input to perform BrotliFillBitWindow. */
static BROTLI_INLINE uint32_t BrotliReadBits(
BrotliBitReader* const br, uint32_t n_bits) {
@@ -321,43 +315,26 @@ static BROTLI_INLINE uint32_t BrotliReadBits(
/* Tries to read the specified amount of bits. Returns 0, if there is not
enough input. n_bits MUST be positive. */
static BROTLI_INLINE int BrotliSafeReadBits(
static BROTLI_INLINE BROTLI_BOOL BrotliSafeReadBits(
BrotliBitReader* const br, uint32_t n_bits, uint32_t* val) {
while (BrotliGetAvailableBits(br) < n_bits) {
if (!BrotliPullByte(br)) {
return 0;
return BROTLI_FALSE;
}
}
BrotliTakeBits(br, n_bits, val);
return 1;
return BROTLI_TRUE;
}
/* Advances the bit reader position to the next byte boundary and verifies
that any skipped bits are set to zero. */
static BROTLI_INLINE int BrotliJumpToByteBoundary(BrotliBitReader* br) {
static BROTLI_INLINE BROTLI_BOOL BrotliJumpToByteBoundary(BrotliBitReader* br) {
uint32_t pad_bits_count = BrotliGetAvailableBits(br) & 0x7;
uint32_t pad_bits = 0;
if (pad_bits_count != 0) {
BrotliTakeBits(br, pad_bits_count, &pad_bits);
}
return pad_bits == 0;
}
/* Peeks a byte at specified offset.
Precondition: bit reader is parked to a byte boundary.
Returns -1 if operation is not feasible. */
static BROTLI_INLINE int BrotliPeekByte(BrotliBitReader* br, size_t offset) {
uint32_t available_bits = BrotliGetAvailableBits(br);
size_t bytes_left = available_bits >> 3;
BROTLI_DCHECK((available_bits & 7) == 0);
if (offset < bytes_left) {
return (BrotliGetBitsUnmasked(br) >> (unsigned)(offset << 3)) & 0xFF;
}
offset -= bytes_left;
if (offset < br->avail_in) {
return br->next_in[offset];
}
return -1;
return TO_BROTLI_BOOL(pad_bits == 0);
}
/* Copies remaining input bytes stored in the bit reader to the output. Value

2
modules/brotli/dec/context.h
View File

@@ -99,7 +99,7 @@
#ifndef BROTLI_DEC_CONTEXT_H_
#define BROTLI_DEC_CONTEXT_H_
#include "./types.h"
#include <brotli/types.h>
enum ContextType {
CONTEXT_LSB6 = 0,

1267
modules/brotli/dec/decode.c
View File

File diff suppressed because it is too large Load Diff

162
modules/brotli/dec/decode.h
View File

@@ -1,162 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* API for Brotli decompression */
#ifndef BROTLI_DEC_DECODE_H_
#define BROTLI_DEC_DECODE_H_
#include "./types.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
typedef struct BrotliStateStruct BrotliState;
typedef enum {
/* Decoding error, e.g. corrupt input or memory allocation problem */
BROTLI_RESULT_ERROR = 0,
/* Decoding successfully completed */
BROTLI_RESULT_SUCCESS = 1,
/* Partially done; should be called again with more input */
BROTLI_RESULT_NEEDS_MORE_INPUT = 2,
/* Partially done; should be called again with more output */
BROTLI_RESULT_NEEDS_MORE_OUTPUT = 3
} BrotliResult;
#define BROTLI_ERROR_CODES_LIST(BROTLI_ERROR_CODE, SEPARATOR) \
BROTLI_ERROR_CODE(_, NO_ERROR, 0) SEPARATOR \
/* Same as BrotliResult values */ \
BROTLI_ERROR_CODE(_, SUCCESS, 1) SEPARATOR \
BROTLI_ERROR_CODE(_, NEEDS_MORE_INPUT, 2) SEPARATOR \
BROTLI_ERROR_CODE(_, NEEDS_MORE_OUTPUT, 3) SEPARATOR \
\
/* Errors caused by invalid input */ \
BROTLI_ERROR_CODE(_ERROR_FORMAT_, EXUBERANT_NIBBLE, -1) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_FORMAT_, RESERVED, -2) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_FORMAT_, EXUBERANT_META_NIBBLE, -3) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_FORMAT_, SIMPLE_HUFFMAN_ALPHABET, -4) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_FORMAT_, SIMPLE_HUFFMAN_SAME, -5) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_FORMAT_, CL_SPACE, -6) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_FORMAT_, HUFFMAN_SPACE, -7) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_FORMAT_, CONTEXT_MAP_REPEAT, -8) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_FORMAT_, BLOCK_LENGTH_1, -9) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_FORMAT_, BLOCK_LENGTH_2, -10) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_FORMAT_, TRANSFORM, -11) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_FORMAT_, DICTIONARY, -12) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_FORMAT_, WINDOW_BITS, -13) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_FORMAT_, PADDING_1, -14) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_FORMAT_, PADDING_2, -15) SEPARATOR \
\
/* -16..-20 codes are reserved */ \
\
/* Memory allocation problems */ \
BROTLI_ERROR_CODE(_ERROR_ALLOC_, CONTEXT_MODES, -21) SEPARATOR \
/* Literal, insert and distance trees together */ \
BROTLI_ERROR_CODE(_ERROR_ALLOC_, TREE_GROUPS, -22) SEPARATOR \
/* -23..-24 codes are reserved for distinct tree groups */ \
BROTLI_ERROR_CODE(_ERROR_ALLOC_, CONTEXT_MAP, -25) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_ALLOC_, RING_BUFFER_1, -26) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_ALLOC_, RING_BUFFER_2, -27) SEPARATOR \
/* -28..-29 codes are reserved for dynamic ringbuffer allocation */ \
BROTLI_ERROR_CODE(_ERROR_ALLOC_, BLOCK_TYPE_TREES, -30) SEPARATOR \
\
/* "Impossible" states */ \
BROTLI_ERROR_CODE(_ERROR_, UNREACHABLE, -31)
typedef enum {
#define _BROTLI_COMMA ,
#define _BROTLI_ERROR_CODE_ENUM_ITEM(PREFIX, NAME, CODE) \
BROTLI ## PREFIX ## NAME = CODE
BROTLI_ERROR_CODES_LIST(_BROTLI_ERROR_CODE_ENUM_ITEM, _BROTLI_COMMA)
#undef _BROTLI_ERROR_CODE_ENUM_ITEM
#undef _BROTLI_COMMA
} BrotliErrorCode;
#define BROTLI_LAST_ERROR_CODE BROTLI_ERROR_UNREACHABLE
/* Creates the instance of BrotliState and initializes it. |alloc_func| and
|free_func| MUST be both zero or both non-zero. In the case they are both
zero, default memory allocators are used. |opaque| is passed to |alloc_func|
and |free_func| when they are called. */
BrotliState* BrotliCreateState(
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
/* Deinitializes and frees BrotliState instance. */
void BrotliDestroyState(BrotliState* state);
/* Sets |*decoded_size| to the decompressed size of the given encoded stream.
This function only works if the encoded buffer has a single meta block,
or if it has two meta-blocks, where the first is uncompressed and the
second is empty.
Returns 1 on success, 0 on failure. */
int BrotliDecompressedSize(size_t encoded_size,
const uint8_t* encoded_buffer,
size_t* decoded_size);
/* Decompresses the data in |encoded_buffer| into |decoded_buffer|, and sets
|*decoded_size| to the decompressed length. */
BrotliResult BrotliDecompressBuffer(size_t encoded_size,
const uint8_t* encoded_buffer,
size_t* decoded_size,
uint8_t* decoded_buffer);
/* Decompresses the data. Supports partial input and output.
Must be called with an allocated input buffer in |*next_in| and an allocated
output buffer in |*next_out|. The values |*available_in| and |*available_out|
must specify the allocated size in |*next_in| and |*next_out| respectively.
After each call, |*available_in| will be decremented by the amount of input
bytes consumed, and the |*next_in| pointer will be incremented by that
amount. Similarly, |*available_out| will be decremented by the amount of
output bytes written, and the |*next_out| pointer will be incremented by that
amount. |total_out|, if it is not a null-pointer, will be set to the number
of bytes decompressed since the last state initialization.
Input is never overconsumed, so |next_in| and |available_in| could be passed
to the next consumer after decoding is complete. */
BrotliResult BrotliDecompressStream(size_t* available_in,
const uint8_t** next_in,
size_t* available_out,
uint8_t** next_out,
size_t* total_out,
BrotliState* s);
/* Fills the new state with a dictionary for LZ77, warming up the ringbuffer,
e.g. for custom static dictionaries for data formats.
Not to be confused with the built-in transformable dictionary of Brotli.
|size| should be less or equal to 2^24 (16MiB), otherwise the dictionary will
be ignored. The dictionary must exist in memory until decoding is done and
is owned by the caller. To use:
1) Allocate and initialize state with BrotliCreateState
2) Use BrotliSetCustomDictionary
3) Use BrotliDecompressStream
4) Clean up and free state with BrotliDestroyState
*/
void BrotliSetCustomDictionary(
size_t size, const uint8_t* dict, BrotliState* s);
/* Returns 1, if s is in a state where we have not read any input bytes yet,
and 0 otherwise */
int BrotliStateIsStreamStart(const BrotliState* s);
/* Returns 1, if s is in a state where we reached the end of the input and
produced all of the output, and 0 otherwise. */
int BrotliStateIsStreamEnd(const BrotliState* s);
/* Returns detailed error code after BrotliDecompressStream returns
BROTLI_RESULT_ERROR. */
BrotliErrorCode BrotliGetErrorCode(const BrotliState* s);
const char* BrotliErrorString(BrotliErrorCode c);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_DEC_DECODE_H_ */

9466
modules/brotli/dec/dictionary.c
View File

File diff suppressed because it is too large Load Diff

38
modules/brotli/dec/dictionary.h
View File

@@ -1,38 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Collection of static dictionary words. */
#ifndef BROTLI_DEC_DICTIONARY_H_
#define BROTLI_DEC_DICTIONARY_H_
#include "./types.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
extern const uint8_t kBrotliDictionary[122784];
static const uint32_t kBrotliDictionaryOffsetsByLength[] = {
0, 0, 0, 0, 0, 4096, 9216, 21504, 35840, 44032, 53248, 63488, 74752, 87040,
93696, 100864, 104704, 106752, 108928, 113536, 115968, 118528, 119872, 121280,
122016
};
static const uint8_t kBrotliDictionarySizeBitsByLength[] = {
0, 0, 0, 0, 10, 10, 11, 11, 10, 10, 10, 10, 10,
9, 9, 8, 7, 7, 8, 7, 7, 6, 6, 5, 5,
};
static const int kBrotliMinDictionaryWordLength = 4;
static const int kBrotliMaxDictionaryWordLength = 24;
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_DEC_DICTIONARY_H_ */

54
modules/brotli/dec/huffman.c
View File

@@ -10,8 +10,9 @@
#include <string.h> /* memcpy, memset */
#include "../common/constants.h"
#include <brotli/types.h>
#include "./port.h"
#include "./types.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -20,7 +21,8 @@ extern "C" {
#define BROTLI_REVERSE_BITS_MAX 8
#ifdef BROTLI_RBIT
#define BROTLI_REVERSE_BITS_BASE (32 - BROTLI_REVERSE_BITS_MAX)
#define BROTLI_REVERSE_BITS_BASE \
((sizeof(reg_t) << 3) - BROTLI_REVERSE_BITS_MAX)
#else
#define BROTLI_REVERSE_BITS_BASE 0
static uint8_t kReverseBits[1 << BROTLI_REVERSE_BITS_MAX] = {
@@ -60,12 +62,12 @@ static uint8_t kReverseBits[1 << BROTLI_REVERSE_BITS_MAX] = {
#endif /* BROTLI_RBIT */
#define BROTLI_REVERSE_BITS_LOWEST \
(1U << (BROTLI_REVERSE_BITS_MAX - 1 + BROTLI_REVERSE_BITS_BASE))
((reg_t)1 << (BROTLI_REVERSE_BITS_MAX - 1 + BROTLI_REVERSE_BITS_BASE))
/* Returns reverse(num >> BROTLI_REVERSE_BITS_BASE, BROTLI_REVERSE_BITS_MAX),
where reverse(value, len) is the bit-wise reversal of the len least
significant bits of value. */
static BROTLI_INLINE uint32_t BrotliReverseBits(uint32_t num) {
static BROTLI_INLINE reg_t BrotliReverseBits(reg_t num) {
#ifdef BROTLI_RBIT
return BROTLI_RBIT(num);
#else
@@ -102,13 +104,13 @@ static BROTLI_INLINE int NextTableBitSize(const uint16_t* const count,
void BrotliBuildCodeLengthsHuffmanTable(HuffmanCode* table,
const uint8_t* const code_lengths,
uint16_t* count) {
HuffmanCode code; /* current table entry */
int symbol; /* symbol index in original or sorted table */
uint32_t key; /* prefix code */
uint32_t key_step; /* prefix code addend */
int step; /* step size to replicate values in current table */
int table_size; /* size of current table */
int sorted[18]; /* symbols sorted by code length */
HuffmanCode code; /* current table entry */
int symbol; /* symbol index in original or sorted table */
reg_t key; /* prefix code */
reg_t key_step; /* prefix code addend */
int step; /* step size to replicate values in current table */
int table_size; /* size of current table */
int sorted[BROTLI_CODE_LENGTH_CODES]; /* symbols sorted by code length */
/* offsets in sorted table for each length */
int offset[BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH + 1];
int bits;
@@ -125,10 +127,10 @@ void BrotliBuildCodeLengthsHuffmanTable(HuffmanCode* table,
bits++;
});
/* Symbols with code length 0 are placed after all other symbols. */
offset[0] = 17;
offset[0] = BROTLI_CODE_LENGTH_CODES - 1;
/* sort symbols by length, by symbol order within each length */
symbol = 18;
symbol = BROTLI_CODE_LENGTH_CODES;
do {
BROTLI_REPEAT(6, {
symbol--;
@@ -142,7 +144,7 @@ void BrotliBuildCodeLengthsHuffmanTable(HuffmanCode* table,
if (offset[0] == 0) {
code.bits = 0;
code.value = (uint16_t)sorted[0];
for (key = 0; key < (uint32_t)table_size; ++key) {
for (key = 0; key < (reg_t)table_size; ++key) {
table[key] = code;
}
return;
@@ -170,18 +172,18 @@ uint32_t BrotliBuildHuffmanTable(HuffmanCode* root_table,
int root_bits,
const uint16_t* const symbol_lists,
uint16_t* count) {
HuffmanCode code; /* current table entry */
HuffmanCode* table; /* next available space in table */
int len; /* current code length */
int symbol; /* symbol index in original or sorted table */
uint32_t key; /* prefix code */
uint32_t key_step; /* prefix code addend */
uint32_t sub_key; /* 2nd level table prefix code */
uint32_t sub_key_step; /* 2nd level table prefix code addend */
int step; /* step size to replicate values in current table */
int table_bits; /* key length of current table */
int table_size; /* size of current table */
int total_size; /* sum of root table size and 2nd level table sizes */
HuffmanCode code; /* current table entry */
HuffmanCode* table; /* next available space in table */
int len; /* current code length */
int symbol; /* symbol index in original or sorted table */
reg_t key; /* prefix code */
reg_t key_step; /* prefix code addend */
reg_t sub_key; /* 2nd level table prefix code */
reg_t sub_key_step; /* 2nd level table prefix code addend */
int step; /* step size to replicate values in current table */
int table_bits; /* key length of current table */
int table_size; /* size of current table */
int total_size; /* sum of root table size and 2nd level table sizes */
int max_length = -1;
int bits;
int bits_count;

8
modules/brotli/dec/huffman.h
View File

@@ -9,7 +9,7 @@
#ifndef BROTLI_DEC_HUFFMAN_H_
#define BROTLI_DEC_HUFFMAN_H_
#include "./types.h"
#include <brotli/types.h>
#include "./port.h"
#if defined(__cplusplus) || defined(c_plusplus)
@@ -18,16 +18,16 @@ extern "C" {
#define BROTLI_HUFFMAN_MAX_CODE_LENGTH 15
/* For current format this constant equals to kNumInsertAndCopyCodes */
#define BROTLI_HUFFMAN_MAX_CODE_LENGTHS_SIZE 704
/* Maximum possible Huffman table size for an alphabet size of (index * 32),
* max code length 15 and root table bits 8. */
static const uint16_t kMaxHuffmanTableSize[] = {
256, 402, 436, 468, 500, 534, 566, 598, 630, 662, 694, 726, 758, 790, 822,
854, 886, 920, 952, 984, 1016, 1048, 1080};
/* BROTLI_NUM_BLOCK_LEN_SYMBOLS == 26 */
#define BROTLI_HUFFMAN_MAX_SIZE_26 396
/* BROTLI_MAX_BLOCK_TYPE_SYMBOLS == 258 */
#define BROTLI_HUFFMAN_MAX_SIZE_258 632
/* BROTLI_MAX_CONTEXT_MAP_SYMBOLS == 272 */
#define BROTLI_HUFFMAN_MAX_SIZE_272 646
#define BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH 5

124
modules/brotli/dec/port.h
View File

@@ -16,6 +16,7 @@
features and attributes
* BROTLI_BUILD_PORTABLE disables dangerous optimizations, like unaligned
read and overlapping memcpy; this reduces decompression speed by 5%
* BROTLI_BUILD_NO_RBIT disables "rbit" optimization for ARM CPUs
* BROTLI_DEBUG dumps file name and line number when decoder detects stream
or memory error
* BROTLI_ENABLE_LOG enables asserts and dumps various state information
@@ -29,27 +30,16 @@
#include <stdio.h>
#endif
/* Compatibility with non-clang compilers. */
#ifndef __has_builtin
#define __has_builtin(x) 0
#endif
#ifndef __has_attribute
#define __has_attribute(x) 0
#endif
#ifndef __has_feature
#define __has_feature(x) 0
#endif
#include <brotli/port.h>
#if defined(__arm__) || defined(__thumb__) || \
defined(_M_ARM) || defined(_M_ARMT)
defined(_M_ARM) || defined(_M_ARMT) || defined(__ARM64_ARCH_8__)
#define BROTLI_TARGET_ARM
#if (defined(__ARM_ARCH) && (__ARM_ARCH >= 7)) || \
(defined(M_ARM) && (M_ARM >= 7))
#if (defined(__ARM_ARCH) && (__ARM_ARCH == 7)) || \
(defined(M_ARM) && (M_ARM == 7))
#define BROTLI_TARGET_ARMV7
#endif /* ARMv7 */
#if defined(__aarch64__)
#if defined(__aarch64__) || defined(__ARM64_ARCH_8__)
#define BROTLI_TARGET_ARMV8
#endif /* ARMv8 */
#endif /* ARM */
@@ -66,61 +56,16 @@
#define BROTLI_TARGET_POWERPC64
#endif
#if defined(__GNUC__) && defined(__GNUC_MINOR__)
#define BROTLI_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
#else
#define BROTLI_GCC_VERSION 0
#endif
#if defined(__ICC)
#define BROTLI_ICC_VERSION __ICC
#else
#define BROTLI_ICC_VERSION 0
#endif
#if defined(BROTLI_BUILD_MODERN_COMPILER)
#define BROTLI_MODERN_COMPILER 1
#elif (BROTLI_GCC_VERSION > 300) || (BROTLI_ICC_VERSION >= 1600)
#define BROTLI_MODERN_COMPILER 1
#else
#define BROTLI_MODERN_COMPILER 0
#endif
#ifdef BROTLI_BUILD_PORTABLE
#define BROTLI_ALIGNED_READ (!!1)
#elif defined(BROTLI_TARGET_X86) || defined(BROTLI_TARGET_X64) || \
defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8)
/* Allow unaligned read only for whitelisted CPUs. */
/* Allow unaligned read only for white-listed CPUs. */
#define BROTLI_ALIGNED_READ (!!0)
#else
#define BROTLI_ALIGNED_READ (!!1)
#endif
/* Define "PREDICT_TRUE" and "PREDICT_FALSE" macros for capable compilers.
To apply compiler hint, enclose the branching condition into macros, like this:
if (PREDICT_TRUE(zero == 0)) {
// main execution path
} else {
// compiler should place this code outside of main execution path
}
OR:
if (PREDICT_FALSE(something_rare_or_unexpected_happens)) {
// compiler should place this code outside of main execution path
}
*/
#if BROTLI_MODERN_COMPILER || __has_builtin(__builtin_expect)
#define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
#define PREDICT_FALSE(x) (__builtin_expect(x, 0))
#else
#define PREDICT_FALSE(x) (x)
#define PREDICT_TRUE(x) (x)
#endif
/* IS_CONSTANT macros returns true for compile-time constant expressions. */
#if BROTLI_MODERN_COMPILER || __has_builtin(__builtin_constant_p)
#define IS_CONSTANT(x) (!!__builtin_constant_p(x))
@@ -128,35 +73,6 @@ OR:
#define IS_CONSTANT(x) (!!0)
#endif
#if BROTLI_MODERN_COMPILER || __has_attribute(always_inline)
#define ATTRIBUTE_ALWAYS_INLINE __attribute__ ((always_inline))
#else
#define ATTRIBUTE_ALWAYS_INLINE
#endif
#if defined(_WIN32) || defined(__CYGWIN__)
#define ATTRIBUTE_VISIBILITY_HIDDEN
#elif BROTLI_MODERN_COMPILER || __has_attribute(visibility)
#define ATTRIBUTE_VISIBILITY_HIDDEN __attribute__ ((visibility ("hidden")))
#else
#define ATTRIBUTE_VISIBILITY_HIDDEN
#endif
#ifndef BROTLI_INTERNAL
#define BROTLI_INTERNAL ATTRIBUTE_VISIBILITY_HIDDEN
#endif
#ifndef _MSC_VER
#if defined(__cplusplus) || !defined(__STRICT_ANSI__) || \
__STDC_VERSION__ >= 199901L
#define BROTLI_INLINE inline ATTRIBUTE_ALWAYS_INLINE
#else
#define BROTLI_INLINE
#endif
#else /* _MSC_VER */
#define BROTLI_INLINE __forceinline
#endif /* _MSC_VER */
#ifdef BROTLI_ENABLE_LOG
#define BROTLI_DCHECK(x) assert(x)
#define BROTLI_LOG(x) printf x
@@ -166,7 +82,7 @@ OR:
#endif
#if defined(BROTLI_DEBUG) || defined(BROTLI_ENABLE_LOG)
static inline void BrotliDump(const char* f, int l, const char* fn) {
static BROTLI_INLINE void BrotliDump(const char* f, int l, const char* fn) {
fprintf(stderr, "%s:%d (%s)\n", f, l, fn);
fflush(stderr);
}
@@ -186,6 +102,12 @@ static inline void BrotliDump(const char* f, int l, const char* fn) {
#define BROTLI_64_BITS 0
#endif
#if (BROTLI_64_BITS)
#define reg_t uint64_t
#else
#define reg_t uint32_t
#endif
#if defined(BROTLI_BUILD_BIG_ENDIAN)
#define BROTLI_LITTLE_ENDIAN 0
#define BROTLI_BIG_ENDIAN 1
@@ -211,22 +133,18 @@ static inline void BrotliDump(const char* f, int l, const char* fn) {
#define BROTLI_LITTLE_ENDIAN 0
#endif
#if BROTLI_MODERN_COMPILER || __has_attribute(noinline)
#define BROTLI_NOINLINE __attribute__((noinline))
#else
#define BROTLI_NOINLINE
#endif
#define BROTLI_REPEAT(N, X) { \
if ((N & 1) != 0) {X;} \
if ((N & 2) != 0) {X; X;} \
if ((N & 4) != 0) {X; X; X; X;} \
}
#if BROTLI_MODERN_COMPILER || defined(__llvm__)
#if defined(BROTLI_TARGET_ARMV7)
static BROTLI_INLINE unsigned BrotliRBit(unsigned input) {
unsigned output;
#if (BROTLI_MODERN_COMPILER || defined(__llvm__)) && \
!defined(BROTLI_BUILD_NO_RBIT)
#if defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8)
/* TODO: detect ARMv6T2 and enable this code for it. */
static BROTLI_INLINE reg_t BrotliRBit(reg_t input) {
reg_t output;
__asm__("rbit %0, %1\n" : "=r"(output) : "r"(input));
return output;
}
@@ -247,6 +165,4 @@ static BROTLI_INLINE unsigned BrotliRBit(unsigned input) {
X = NULL; \
}
#define BROTLI_UNUSED(X) (void)(X)
#endif /* BROTLI_DEC_PORT_H_ */

8
modules/brotli/dec/prefix.h
View File

@@ -11,7 +11,8 @@
#ifndef BROTLI_DEC_PREFIX_H_
#define BROTLI_DEC_PREFIX_H_
#include "./types.h"
#include "../common/constants.h"
#include <brotli/types.h>
/* Represents the range of values belonging to a prefix code: */
/* [offset, offset + 2^nbits) */
@@ -20,7 +21,8 @@ struct PrefixCodeRange {
uint8_t nbits;
};
static const struct PrefixCodeRange kBlockLengthPrefixCode[] = {
static const struct PrefixCodeRange
kBlockLengthPrefixCode[BROTLI_NUM_BLOCK_LEN_SYMBOLS] = {
{ 1, 2}, { 5, 2}, { 9, 2}, { 13, 2},
{ 17, 3}, { 25, 3}, { 33, 3}, { 41, 3},
{ 49, 4}, { 65, 4}, { 81, 4}, { 97, 4},
@@ -39,7 +41,7 @@ typedef struct CmdLutElement {
uint16_t copy_len_offset;
} CmdLutElement;
static const CmdLutElement kCmdLut[704] = {
static const CmdLutElement kCmdLut[BROTLI_NUM_COMMAND_SYMBOLS] = {
{ 0x00, 0x00, 0, 0x00, 0x0000, 0x0002 },
{ 0x00, 0x00, 0, 0x01, 0x0000, 0x0003 },
{ 0x00, 0x00, 0, 0x02, 0x0000, 0x0004 },

61
modules/brotli/dec/state.c
View File

@@ -8,17 +8,13 @@
#include <stdlib.h> /* free, malloc */
#include <brotli/types.h>
#include "./huffman.h"
#include "./types.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
/* Declared in decode.h */
int BrotliStateIsStreamStart(const BrotliState* s);
int BrotliStateIsStreamEnd(const BrotliState* s);
static void* DefaultAllocFunc(void* opaque, size_t size) {
BROTLI_UNUSED(opaque);
return malloc(size);
@@ -29,11 +25,11 @@ static void DefaultFreeFunc(void* opaque, void* address) {
free(address);
}
void BrotliStateInit(BrotliState* s) {
BrotliStateInitWithCustomAllocators(s, 0, 0, 0);
void BrotliDecoderStateInit(BrotliDecoderState* s) {
BrotliDecoderStateInitWithCustomAllocators(s, 0, 0, 0);
}
void BrotliStateInitWithCustomAllocators(BrotliState* s,
void BrotliDecoderStateInitWithCustomAllocators(BrotliDecoderState* s,
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque) {
if (!alloc_func) {
s->alloc_func = DefaultAllocFunc;
@@ -55,6 +51,8 @@ void BrotliStateInitWithCustomAllocators(BrotliState* s,
s->substate_decode_uint8 = BROTLI_STATE_DECODE_UINT8_NONE;
s->substate_read_block_length = BROTLI_STATE_READ_BLOCK_LENGTH_NONE;
s->dictionary = BrotliGetDictionary();
s->buffer_length = 0;
s->loop_counter = 0;
s->pos = 0;
@@ -64,6 +62,9 @@ void BrotliStateInitWithCustomAllocators(BrotliState* s,
s->block_type_trees = NULL;
s->block_len_trees = NULL;
s->ringbuffer = NULL;
s->ringbuffer_size = 0;
s->new_ringbuffer_size = 0;
s->ringbuffer_mask = 0;
s->context_map = NULL;
s->context_modes = NULL;
@@ -84,6 +85,7 @@ void BrotliStateInitWithCustomAllocators(BrotliState* s,
s->custom_dict_size = 0;
s->is_last_metablock = 0;
s->should_wrap_ringbuffer = 0;
s->window_bits = 0;
s->max_distance = 0;
s->dist_rb[0] = 16;
@@ -97,10 +99,10 @@ void BrotliStateInitWithCustomAllocators(BrotliState* s,
/* Make small negative indexes addressable. */
s->symbol_lists = &s->symbols_lists_array[BROTLI_HUFFMAN_MAX_CODE_LENGTH + 1];
s->mtf_upper_bound = 255;
s->mtf_upper_bound = 63;
}
void BrotliStateMetablockBegin(BrotliState* s) {
void BrotliDecoderStateMetablockBegin(BrotliDecoderState* s) {
s->meta_block_remaining_len = 0;
s->block_length[0] = 1U << 28;
s->block_length[1] = 1U << 28;
@@ -131,48 +133,35 @@ void BrotliStateMetablockBegin(BrotliState* s) {
s->distance_hgroup.htrees = NULL;
}
void BrotliStateCleanupAfterMetablock(BrotliState* s) {
void BrotliDecoderStateCleanupAfterMetablock(BrotliDecoderState* s) {
BROTLI_FREE(s, s->context_modes);
BROTLI_FREE(s, s->context_map);
BROTLI_FREE(s, s->dist_context_map);
BrotliHuffmanTreeGroupRelease(s, &s->literal_hgroup);
BrotliHuffmanTreeGroupRelease(s, &s->insert_copy_hgroup);
BrotliHuffmanTreeGroupRelease(s, &s->distance_hgroup);
BROTLI_FREE(s, s->literal_hgroup.htrees);
BROTLI_FREE(s, s->insert_copy_hgroup.htrees);
BROTLI_FREE(s, s->distance_hgroup.htrees);
}
void BrotliStateCleanup(BrotliState* s) {
BrotliStateCleanupAfterMetablock(s);
void BrotliDecoderStateCleanup(BrotliDecoderState* s) {
BrotliDecoderStateCleanupAfterMetablock(s);
BROTLI_FREE(s, s->ringbuffer);
BROTLI_FREE(s, s->block_type_trees);
}
int BrotliStateIsStreamStart(const BrotliState* s) {
return (s->state == BROTLI_STATE_UNINITED &&
BrotliGetAvailableBits(&s->br) == 0);
}
int BrotliStateIsStreamEnd(const BrotliState* s) {
return s->state == BROTLI_STATE_DONE;
}
void BrotliHuffmanTreeGroupInit(BrotliState* s, HuffmanTreeGroup* group,
uint32_t alphabet_size, uint32_t ntrees) {
BROTLI_BOOL BrotliDecoderHuffmanTreeGroupInit(BrotliDecoderState* s,
HuffmanTreeGroup* group, uint32_t alphabet_size, uint32_t ntrees) {
/* Pack two allocations into one */
const size_t max_table_size = kMaxHuffmanTableSize[(alphabet_size + 31) >> 5];
const size_t code_size = sizeof(HuffmanCode) * ntrees * max_table_size;
const size_t htree_size = sizeof(HuffmanCode*) * ntrees;
char* p = (char*)BROTLI_ALLOC(s, code_size + htree_size);
/* Pointer alignment is, hopefully, wider than sizeof(HuffmanCode). */
HuffmanCode** p = (HuffmanCode**)BROTLI_ALLOC(s, code_size + htree_size);
group->alphabet_size = (uint16_t)alphabet_size;
group->num_htrees = (uint16_t)ntrees;
group->codes = (HuffmanCode*)p;
group->htrees = (HuffmanCode**)(p + code_size);
}
void BrotliHuffmanTreeGroupRelease(BrotliState* s, HuffmanTreeGroup* group) {
BROTLI_FREE(s, group->codes);
group->htrees = NULL;
group->htrees = p;
group->codes = (HuffmanCode*)(&p[ntrees]);
return !!p;
}
#if defined(__cplusplus) || defined(c_plusplus)

56
modules/brotli/dec/state.h
View File

@@ -9,9 +9,11 @@
#ifndef BROTLI_DEC_STATE_H_
#define BROTLI_DEC_STATE_H_
#include "../common/constants.h"
#include "../common/dictionary.h"
#include <brotli/types.h>
#include "./bit_reader.h"
#include "./huffman.h"
#include "./types.h"
#include "./port.h"
#if defined(__cplusplus) || defined(c_plusplus)
@@ -93,7 +95,7 @@ typedef enum {
BROTLI_STATE_READ_BLOCK_LENGTH_SUFFIX
} BrotliRunningReadBlockLengthState;
struct BrotliStateStruct {
struct BrotliDecoderStateStruct {
BrotliRunningState state;
/* This counter is reused for several disjoint loops. */
@@ -114,7 +116,6 @@ struct BrotliStateStruct {
int pos;
int max_backward_distance;
int max_backward_distance_minus_custom_dict_size;
int max_distance;
int ringbuffer_size;
int ringbuffer_mask;
@@ -140,6 +141,8 @@ struct BrotliStateStruct {
/* This is true if the literal context map histogram type always matches the
block type. It is then not needed to keep the context (faster decoding). */
int trivial_literal_context;
/* Distance context is actual after command is decoded and before distance
is computed. After distance computation it is used as a temporary variable. */
int distance_context;
int meta_block_remaining_len;
uint32_t block_length_index;
@@ -160,8 +163,8 @@ struct BrotliStateStruct {
int distance_code;
/* For partial write operations */
size_t rb_roundtrips; /* How many times we went around the ringbuffer */
size_t partial_pos_out; /* How much output to the user in total (<= rb) */
size_t rb_roundtrips; /* How many times we went around the ring-buffer */
size_t partial_pos_out; /* How much output to the user in total */
/* For ReadHuffmanCode */
uint32_t symbol;
@@ -173,10 +176,10 @@ struct BrotliStateStruct {
uint16_t* symbol_lists;
/* Storage from symbol_lists. */
uint16_t symbols_lists_array[BROTLI_HUFFMAN_MAX_CODE_LENGTH + 1 +
BROTLI_HUFFMAN_MAX_CODE_LENGTHS_SIZE];
BROTLI_NUM_COMMAND_SYMBOLS];
/* Tails of symbol chains. */
int next_symbol[32];
uint8_t code_length_code_lengths[18];
uint8_t code_length_code_lengths[BROTLI_CODE_LENGTH_CODES];
/* Population counts for the code lengths */
uint16_t code_length_histo[16];
@@ -192,7 +195,7 @@ struct BrotliStateStruct {
/* For InverseMoveToFrontTransform */
uint32_t mtf_upper_bound;
uint8_t mtf[256 + 4];
uint32_t mtf[64 + 1];
/* For custom dictionaries */
const uint8_t* custom_dict;
@@ -208,32 +211,37 @@ struct BrotliStateStruct {
BrotliRunningDecodeUint8State substate_decode_uint8;
BrotliRunningReadBlockLengthState substate_read_block_length;
uint8_t is_last_metablock;
uint8_t is_uncompressed;
uint8_t is_metadata;
uint8_t size_nibbles;
unsigned int is_last_metablock : 1;
unsigned int is_uncompressed : 1;
unsigned int is_metadata : 1;
unsigned int should_wrap_ringbuffer : 1;
unsigned int size_nibbles : 8;
uint32_t window_bits;
int new_ringbuffer_size;
uint32_t num_literal_htrees;
uint8_t* context_map;
uint8_t* context_modes;
const BrotliDictionary* dictionary;
uint32_t trivial_literal_contexts[8]; /* 256 bits */
};
typedef struct BrotliStateStruct BrotliStateInternal;
#define BrotliState BrotliStateInternal
typedef struct BrotliDecoderStateStruct BrotliDecoderStateInternal;
#define BrotliDecoderState BrotliDecoderStateInternal
BROTLI_INTERNAL void BrotliStateInit(BrotliState* s);
BROTLI_INTERNAL void BrotliStateInitWithCustomAllocators(BrotliState* s,
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
BROTLI_INTERNAL void BrotliStateCleanup(BrotliState* s);
BROTLI_INTERNAL void BrotliStateMetablockBegin(BrotliState* s);
BROTLI_INTERNAL void BrotliStateCleanupAfterMetablock(BrotliState* s);
BROTLI_INTERNAL void BrotliHuffmanTreeGroupInit(BrotliState* s,
HuffmanTreeGroup* group, uint32_t alphabet_size, uint32_t ntrees);
BROTLI_INTERNAL void BrotliHuffmanTreeGroupRelease(BrotliState* s,
HuffmanTreeGroup* group);
BROTLI_INTERNAL void BrotliDecoderStateInit(BrotliDecoderState* s);
BROTLI_INTERNAL void BrotliDecoderStateInitWithCustomAllocators(
BrotliDecoderState* s, brotli_alloc_func alloc_func,
brotli_free_func free_func, void* opaque);
BROTLI_INTERNAL void BrotliDecoderStateCleanup(BrotliDecoderState* s);
BROTLI_INTERNAL void BrotliDecoderStateMetablockBegin(BrotliDecoderState* s);
BROTLI_INTERNAL void BrotliDecoderStateCleanupAfterMetablock(
BrotliDecoderState* s);
BROTLI_INTERNAL BROTLI_BOOL BrotliDecoderHuffmanTreeGroupInit(
BrotliDecoderState* s, HuffmanTreeGroup* group, uint32_t alphabet_size,
uint32_t ntrees);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */

4
modules/brotli/dec/transform.h
View File

@@ -9,8 +9,8 @@
#ifndef BROTLI_DEC_TRANSFORM_H_
#define BROTLI_DEC_TRANSFORM_H_
#include <brotli/types.h>
#include "./port.h"
#include "./types.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -247,7 +247,7 @@ static int ToUpperCase(uint8_t* p) {
}
return 1;
}
/* An overly simplified uppercasing model for utf-8. */
/* An overly simplified uppercasing model for UTF-8. */
if (p[0] < 0xe0) {
p[1] ^= 32;
return 2;

38
modules/brotli/dec/types.h
View File

@@ -1,38 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Common types */
#ifndef BROTLI_DEC_TYPES_H_
#define BROTLI_DEC_TYPES_H_
#include <stddef.h> /* for size_t */
#if defined(_MSC_VER) && (_MSC_VER < 1600)
typedef __int8 int8_t;
typedef unsigned __int8 uint8_t;
typedef __int16 int16_t;
typedef unsigned __int16 uint16_t;
typedef __int32 int32_t;
typedef unsigned __int32 uint32_t;
typedef unsigned __int64 uint64_t;
typedef __int64 int64_t;
#else
#include <stdint.h>
#endif /* defined(_MSC_VER) && (_MSC_VER < 1600) */
/* Allocating function pointer. Function MUST return 0 in the case of failure.
Otherwise it MUST return a valid pointer to a memory region of at least
size length. Neither items nor size are allowed to be 0.
opaque argument is a pointer provided by client and could be used to bind
function to specific object (memory pool). */
typedef void* (*brotli_alloc_func)(void* opaque, size_t size);
/* Deallocating function pointer. Function SHOULD be no-op in the case the
address is 0. */
typedef void (*brotli_free_func)(void* opaque, void* address);
#endif /* BROTLI_DEC_TYPES_H_ */

14
modules/brotli/enc/Makefile
View File

@@ -1,14 +0,0 @@
#brotli/enc
include ../shared.mk
OBJS_NODICT = backward_references.o block_splitter.o brotli_bit_stream.o compress_fragment.o compress_fragment_two_pass.o encode.o encode_parallel.o entropy_encode.o histogram.o literal_cost.o metablock.o static_dict.o streams.o utf8_util.o
OBJS = $(OBJS_NODICT) dictionary.o
nodict : $(OBJS_NODICT)
all : $(OBJS)
clean :
rm -f $(OBJS) $(SO)

130
modules/brotli/enc/backward_references.c Normal file
View File

@@ -0,0 +1,130 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Function to find backward reference copies. */
#include "./backward_references.h"
#include "../common/constants.h"
#include "../common/dictionary.h"
#include <brotli/types.h>
#include "./command.h"
#include "./dictionary_hash.h"
#include "./memory.h"
#include "./port.h"
#include "./quality.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance,
size_t max_distance,
const int* dist_cache) {
if (distance <= max_distance) {
size_t distance_plus_3 = distance + 3;
size_t offset0 = distance_plus_3 - (size_t)dist_cache[0];
size_t offset1 = distance_plus_3 - (size_t)dist_cache[1];
if (distance == (size_t)dist_cache[0]) {
return 0;
} else if (distance == (size_t)dist_cache[1]) {
return 1;
} else if (offset0 < 7) {
return (0x9750468 >> (4 * offset0)) & 0xF;
} else if (offset1 < 7) {
return (0xFDB1ACE >> (4 * offset1)) & 0xF;
} else if (distance == (size_t)dist_cache[2]) {
return 2;
} else if (distance == (size_t)dist_cache[3]) {
return 3;
}
}
return distance + BROTLI_NUM_DISTANCE_SHORT_CODES - 1;
}
#define EXPAND_CAT(a, b) CAT(a, b)
#define CAT(a, b) a ## b
#define FN(X) EXPAND_CAT(X, HASHER())
#define HASHER() H2
/* NOLINTNEXTLINE(build/include) */
#include "./backward_references_inc.h"
#undef HASHER
#define HASHER() H3
/* NOLINTNEXTLINE(build/include) */
#include "./backward_references_inc.h"
#undef HASHER
#define HASHER() H4
/* NOLINTNEXTLINE(build/include) */
#include "./backward_references_inc.h"
#undef HASHER
#define HASHER() H5
/* NOLINTNEXTLINE(build/include) */
#include "./backward_references_inc.h"
#undef HASHER
#define HASHER() H6
/* NOLINTNEXTLINE(build/include) */
#include "./backward_references_inc.h"
#undef HASHER
#define HASHER() H40
/* NOLINTNEXTLINE(build/include) */
#include "./backward_references_inc.h"
#undef HASHER
#define HASHER() H41
/* NOLINTNEXTLINE(build/include) */
#include "./backward_references_inc.h"
#undef HASHER
#define HASHER() H42
/* NOLINTNEXTLINE(build/include) */
#include "./backward_references_inc.h"
#undef HASHER
#define HASHER() H54
/* NOLINTNEXTLINE(build/include) */
#include "./backward_references_inc.h"
#undef HASHER
#undef FN
#undef CAT
#undef EXPAND_CAT
void BrotliCreateBackwardReferences(const BrotliDictionary* dictionary,
size_t num_bytes,
size_t position,
const uint8_t* ringbuffer,
size_t ringbuffer_mask,
const BrotliEncoderParams* params,
HasherHandle hasher,
int* dist_cache,
size_t* last_insert_len,
Command* commands,
size_t* num_commands,
size_t* num_literals) {
switch (params->hasher.type) {
#define CASE_(N) \
case N: \
CreateBackwardReferencesH ## N(dictionary, \
kStaticDictionaryHash, num_bytes, position, ringbuffer, \
ringbuffer_mask, params, hasher, dist_cache, \
last_insert_len, commands, num_commands, num_literals); \
break;
FOR_GENERIC_HASHERS(CASE_)
#undef CASE_
default:
break;
}
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

858
modules/brotli/enc/backward_references.cc
View File

@@ -1,858 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
// Function to find backward reference copies.
#include "./backward_references.h"
#include <algorithm>
#include <limits>
#include <vector>
#include "./command.h"
#include "./fast_log.h"
#include "./literal_cost.h"
namespace brotli {
// The maximum length for which the zopflification uses distinct distances.
static const uint16_t kMaxZopfliLen = 325;
// Histogram based cost model for zopflification.
class ZopfliCostModel {
public:
ZopfliCostModel(void) : min_cost_cmd_(kInfinity) {}
void SetFromCommands(size_t num_bytes,
size_t position,
const uint8_t* ringbuffer,
size_t ringbuffer_mask,
const Command* commands,
size_t num_commands,
size_t last_insert_len) {
std::vector<uint32_t> histogram_literal(256, 0);
std::vector<uint32_t> histogram_cmd(kNumCommandPrefixes, 0);
std::vector<uint32_t> histogram_dist(kNumDistancePrefixes, 0);
size_t pos = position - last_insert_len;
for (size_t i = 0; i < num_commands; i++) {
size_t inslength = commands[i].insert_len_;
size_t copylength = commands[i].copy_len();
size_t distcode = commands[i].dist_prefix_;
size_t cmdcode = commands[i].cmd_prefix_;
histogram_cmd[cmdcode]++;
if (cmdcode >= 128) histogram_dist[distcode]++;
for (size_t j = 0; j < inslength; j++) {
histogram_literal[ringbuffer[(pos + j) & ringbuffer_mask]]++;
}
pos += inslength + copylength;
}
std::vector<float> cost_literal;
Set(histogram_literal, &cost_literal);
Set(histogram_cmd, &cost_cmd_);
Set(histogram_dist, &cost_dist_);
for (uint32_t i = 0; i < kNumCommandPrefixes; ++i) {
min_cost_cmd_ = std::min(min_cost_cmd_, cost_cmd_[i]);
}
literal_costs_.resize(num_bytes + 1);
literal_costs_[0] = 0.0;
for (size_t i = 0; i < num_bytes; ++i) {
literal_costs_[i + 1] = literal_costs_[i] +
cost_literal[ringbuffer[(position + i) & ringbuffer_mask]];
}
}
void SetFromLiteralCosts(size_t num_bytes,
size_t position,
const uint8_t* ringbuffer,
size_t ringbuffer_mask) {
literal_costs_.resize(num_bytes + 2);
EstimateBitCostsForLiterals(position, num_bytes, ringbuffer_mask,
ringbuffer, &literal_costs_[1]);
literal_costs_[0] = 0.0;
for (size_t i = 0; i < num_bytes; ++i) {
literal_costs_[i + 1] += literal_costs_[i];
}
cost_cmd_.resize(kNumCommandPrefixes);
cost_dist_.resize(kNumDistancePrefixes);
for (uint32_t i = 0; i < kNumCommandPrefixes; ++i) {
cost_cmd_[i] = static_cast<float>(FastLog2(11 + i));
}
for (uint32_t i = 0; i < kNumDistancePrefixes; ++i) {
cost_dist_[i] = static_cast<float>(FastLog2(20 + i));
}
min_cost_cmd_ = static_cast<float>(FastLog2(11));
}
float GetCommandCost(
size_t dist_code, size_t length_code, size_t insert_length) const {
uint16_t inscode = GetInsertLengthCode(insert_length);
uint16_t copycode = GetCopyLengthCode(length_code);
uint16_t cmdcode = CombineLengthCodes(inscode, copycode, dist_code == 0);
uint16_t dist_symbol;
uint32_t distextra;
PrefixEncodeCopyDistance(dist_code, 0, 0, &dist_symbol, &distextra);
uint32_t distnumextra = distextra >> 24;
float result = static_cast<float>(
GetInsertExtra(inscode) + GetCopyExtra(copycode) + distnumextra);
result += cost_cmd_[cmdcode];
if (cmdcode >= 128) result += cost_dist_[dist_symbol];
return result;
}
float GetLiteralCosts(size_t from, size_t to) const {
return literal_costs_[to] - literal_costs_[from];
}
float GetMinCostCmd(void) const {
return min_cost_cmd_;
}
private:
void Set(const std::vector<uint32_t>& histogram, std::vector<float>* cost) {
cost->resize(histogram.size());
size_t sum = 0;
for (size_t i = 0; i < histogram.size(); i++) {
sum += histogram[i];
}
float log2sum = static_cast<float>(FastLog2(sum));
for (size_t i = 0; i < histogram.size(); i++) {
if (histogram[i] == 0) {
(*cost)[i] = log2sum + 2;
continue;
}
// Shannon bits for this symbol.
(*cost)[i] = log2sum - static_cast<float>(FastLog2(histogram[i]));
// Cannot be coded with less than 1 bit
if ((*cost)[i] < 1) (*cost)[i] = 1;
}
}
std::vector<float> cost_cmd_; // The insert and copy length symbols.
std::vector<float> cost_dist_;
// Cumulative costs of literals per position in the stream.
std::vector<float> literal_costs_;
float min_cost_cmd_;
};
inline size_t ComputeDistanceCode(size_t distance,
size_t max_distance,
int quality,
const int* dist_cache) {
if (distance <= max_distance) {
if (distance == static_cast<size_t>(dist_cache[0])) {
return 0;
} else if (distance == static_cast<size_t>(dist_cache[1])) {
return 1;
} else if (distance == static_cast<size_t>(dist_cache[2])) {
return 2;
} else if (distance == static_cast<size_t>(dist_cache[3])) {
return 3;
} else if (quality > 3 && distance >= 6) {
for (size_t k = 4; k < kNumDistanceShortCodes; ++k) {
size_t idx = kDistanceCacheIndex[k];
size_t candidate =
static_cast<size_t>(dist_cache[idx] + kDistanceCacheOffset[k]);
static const size_t kLimits[16] = { 0, 0, 0, 0,
6, 6, 11, 11,
11, 11, 11, 11,
12, 12, 12, 12 };
if (distance == candidate && distance >= kLimits[k]) {
return k;
}
}
}
}
return distance + 15;
}
// REQUIRES: len >= 2, start_pos <= pos
// REQUIRES: cost < kInfinity, nodes[start_pos].cost < kInfinity
// Maintains the "ZopfliNode array invariant".
inline void UpdateZopfliNode(ZopfliNode* nodes, size_t pos, size_t start_pos,
size_t len, size_t len_code, size_t dist,
size_t short_code, float cost) {
ZopfliNode& next = nodes[pos + len];
next.length = static_cast<uint32_t>(len | ((len + 9u - len_code) << 24));
next.distance = static_cast<uint32_t>(dist | (short_code << 25));
next.insert_length = static_cast<uint32_t>(pos - start_pos);
next.cost = cost;
}
// Maintains the smallest 2^k cost difference together with their positions
class StartPosQueue {
public:
struct PosData {
size_t pos;
int distance_cache[4];
float costdiff;
};
explicit StartPosQueue(int bits)
: mask_((1u << bits) - 1), q_(1 << bits), idx_(0) {}
void Clear(void) {
idx_ = 0;
}
void Push(const StartPosQueue::PosData& posdata) {
size_t offset = ~idx_ & mask_;
++idx_;
size_t len = size();
q_[offset] = posdata;
/* Restore the sorted order. In the list of |len| items at most |len - 1|
adjacent element comparisons / swaps are required. */
for (size_t i = 1; i < len; ++i) {
if (q_[offset & mask_].costdiff > q_[(offset + 1) & mask_].costdiff) {
std::swap(q_[offset & mask_], q_[(offset + 1) & mask_]);
}
++offset;
}
}
size_t size(void) const { return std::min(idx_, mask_ + 1); }
const StartPosQueue::PosData& GetStartPosData(size_t k) const {
return q_[(k - idx_) & mask_];
}
private:
const size_t mask_;
std::vector<PosData> q_;
size_t idx_;
};
// Returns the minimum possible copy length that can improve the cost of any
// future position.
static size_t ComputeMinimumCopyLength(const StartPosQueue& queue,
const ZopfliNode* nodes,
const ZopfliCostModel& model,
const size_t num_bytes,
const size_t pos) {
// Compute the minimum possible cost of reaching any future position.
const size_t start0 = queue.GetStartPosData(0).pos;
float min_cost = (nodes[start0].cost +
model.GetLiteralCosts(start0, pos) +
model.GetMinCostCmd());
size_t len = 2;
size_t next_len_bucket = 4;
size_t next_len_offset = 10;
while (pos + len <= num_bytes && nodes[pos + len].cost <= min_cost) {
// We already reached (pos + len) with no more cost than the minimum
// possible cost of reaching anything from this pos, so there is no point in
// looking for lengths <= len.
++len;
if (len == next_len_offset) {
// We reached the next copy length code bucket, so we add one more
// extra bit to the minimum cost.
min_cost += static_cast<float>(1.0);
next_len_offset += next_len_bucket;
next_len_bucket *= 2;
}
}
return len;
}
// Fills in dist_cache[0..3] with the last four distances (as defined by
// Section 4. of the Spec) that would be used at (block_start + pos) if we
// used the shortest path of commands from block_start, computed from
// nodes[0..pos]. The last four distances at block_start are in
// starting_dist_cach[0..3].
// REQUIRES: nodes[pos].cost < kInfinity
// REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant".
static void ComputeDistanceCache(const size_t block_start,
const size_t pos,
const size_t max_backward,
const int* starting_dist_cache,
const ZopfliNode* nodes,
int* dist_cache) {
int idx = 0;
size_t p = pos;
// Because of prerequisite, does at most (pos + 1) / 2 iterations.
while (idx < 4 && p > 0) {
const size_t clen = nodes[p].copy_length();
const size_t ilen = nodes[p].insert_length;
const size_t dist = nodes[p].copy_distance();
// Since block_start + p is the end position of the command, the copy part
// starts from block_start + p - clen. Distances that are greater than this
// or greater than max_backward are static dictionary references, and do
// not update the last distances. Also distance code 0 (last distance)
// does not update the last distances.
if (dist + clen <= block_start + p && dist <= max_backward &&
nodes[p].distance_code() > 0) {
dist_cache[idx++] = static_cast<int>(dist);
}
// Because of prerequisite, p >= clen + ilen >= 2.
p -= clen + ilen;
}
for (; idx < 4; ++idx) {
dist_cache[idx] = *starting_dist_cache++;
}
}
static void UpdateNodes(const size_t num_bytes,
const size_t block_start,
const size_t pos,
const uint8_t* ringbuffer,
const size_t ringbuffer_mask,
const size_t max_backward_limit,
const int* starting_dist_cache,
const size_t num_matches,
const BackwardMatch* matches,
const ZopfliCostModel* model,
StartPosQueue* queue,
ZopfliNode* nodes) {
size_t cur_ix = block_start + pos;
size_t cur_ix_masked = cur_ix & ringbuffer_mask;
size_t max_distance = std::min(cur_ix, max_backward_limit);
if (nodes[pos].cost <= model->GetLiteralCosts(0, pos)) {
StartPosQueue::PosData posdata;
posdata.pos = pos;
posdata.costdiff = nodes[pos].cost - model->GetLiteralCosts(0, pos);
ComputeDistanceCache(block_start, pos, max_backward_limit,
starting_dist_cache, nodes, posdata.distance_cache);
queue->Push(posdata);
}
const size_t min_len = ComputeMinimumCopyLength(
*queue, nodes, *model, num_bytes, pos);
// Go over the command starting positions in order of increasing cost
// difference.
for (size_t k = 0; k < 5 && k < queue->size(); ++k) {
const StartPosQueue::PosData& posdata = queue->GetStartPosData(k);
const size_t start = posdata.pos;
const float start_costdiff = posdata.costdiff;
// Look for last distance matches using the distance cache from this
// starting position.
size_t best_len = min_len - 1;
for (size_t j = 0; j < kNumDistanceShortCodes; ++j) {
const size_t idx = kDistanceCacheIndex[j];
const size_t backward = static_cast<size_t>(posdata.distance_cache[idx] +
kDistanceCacheOffset[j]);
size_t prev_ix = cur_ix - backward;
if (prev_ix >= cur_ix) {
continue;
}
if (PREDICT_FALSE(backward > max_distance)) {
continue;
}
prev_ix &= ringbuffer_mask;
if (cur_ix_masked + best_len > ringbuffer_mask ||
prev_ix + best_len > ringbuffer_mask ||
ringbuffer[cur_ix_masked + best_len] !=
ringbuffer[prev_ix + best_len]) {
continue;
}
const size_t len =
FindMatchLengthWithLimit(&ringbuffer[prev_ix],
&ringbuffer[cur_ix_masked],
num_bytes - pos);
for (size_t l = best_len + 1; l <= len; ++l) {
const size_t inslen = pos - start;
float cmd_cost = model->GetCommandCost(j, l, inslen);
float cost = start_costdiff + cmd_cost + model->GetLiteralCosts(0, pos);
if (cost < nodes[pos + l].cost) {
UpdateZopfliNode(&nodes[0], pos, start, l, l, backward, j + 1, cost);
}
best_len = l;
}
}
// At higher iterations look only for new last distance matches, since
// looking only for new command start positions with the same distances
// does not help much.
if (k >= 2) continue;
// Loop through all possible copy lengths at this position.
size_t len = min_len;
for (size_t j = 0; j < num_matches; ++j) {
BackwardMatch match = matches[j];
size_t dist = match.distance;
bool is_dictionary_match = dist > max_distance;
// We already tried all possible last distance matches, so we can use
// normal distance code here.
size_t dist_code = dist + 15;
// Try all copy lengths up until the maximum copy length corresponding
// to this distance. If the distance refers to the static dictionary, or
// the maximum length is long enough, try only one maximum length.
size_t max_len = match.length();
if (len < max_len && (is_dictionary_match || max_len > kMaxZopfliLen)) {
len = max_len;
}
for (; len <= max_len; ++len) {
size_t len_code = is_dictionary_match ? match.length_code() : len;
const size_t inslen = pos - start;
float cmd_cost = model->GetCommandCost(dist_code, len_code, inslen);
float cost = start_costdiff + cmd_cost + model->GetLiteralCosts(0, pos);
if (cost < nodes[pos + len].cost) {
UpdateZopfliNode(&nodes[0], pos, start, len, len_code, dist, 0, cost);
}
}
}
}
}
static void ComputeShortestPathFromNodes(size_t num_bytes,
const ZopfliNode* nodes,
std::vector<uint32_t>* path) {
std::vector<uint32_t> backwards(num_bytes / 2 + 1);
size_t index = num_bytes;
while (nodes[index].cost == kInfinity) --index;
size_t num_commands = 0;
while (index != 0) {
size_t len = nodes[index].command_length();
backwards[num_commands++] = static_cast<uint32_t>(len);
index -= len;
}
path->resize(num_commands);
for (size_t i = num_commands, j = 0; i > 0; --i, ++j) {
(*path)[j] = backwards[i - 1];
}
}
void ZopfliCreateCommands(const size_t num_bytes,
const size_t block_start,
const size_t max_backward_limit,
const std::vector<uint32_t>& path,
const ZopfliNode* nodes,
int* dist_cache,
size_t* last_insert_len,
Command* commands,
size_t* num_literals) {
size_t pos = 0;
for (size_t i = 0; i < path.size(); i++) {
const ZopfliNode& next = nodes[pos + path[i]];
size_t copy_length = next.copy_length();
size_t insert_length = next.insert_length;
pos += insert_length;
if (i == 0) {
insert_length += *last_insert_len;
*last_insert_len = 0;
}
size_t distance = next.copy_distance();
size_t len_code = next.length_code();
size_t max_distance = std::min(block_start + pos, max_backward_limit);
bool is_dictionary = (distance > max_distance);
size_t dist_code = next.distance_code();
Command cmd(insert_length, copy_length, len_code, dist_code);
commands[i] = cmd;
if (!is_dictionary && dist_code > 0) {
dist_cache[3] = dist_cache[2];
dist_cache[2] = dist_cache[1];
dist_cache[1] = dist_cache[0];
dist_cache[0] = static_cast<int>(distance);
}
*num_literals += insert_length;
pos += copy_length;
}
*last_insert_len += num_bytes - pos;
}
static void ZopfliIterate(size_t num_bytes,
size_t position,
const uint8_t* ringbuffer,
size_t ringbuffer_mask,
const size_t max_backward_limit,
const int* dist_cache,
const ZopfliCostModel& model,
const std::vector<uint32_t>& num_matches,
const std::vector<BackwardMatch>& matches,
ZopfliNode* nodes,
std::vector<uint32_t>* path) {
nodes[0].length = 0;
nodes[0].cost = 0;
StartPosQueue queue(3);
size_t cur_match_pos = 0;
for (size_t i = 0; i + 3 < num_bytes; i++) {
UpdateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask,
max_backward_limit, dist_cache, num_matches[i],
&matches[cur_match_pos], &model, &queue, &nodes[0]);
cur_match_pos += num_matches[i];
// The zopflification can be too slow in case of very long lengths, so in
// such case skip it all, it does not cost a lot of compression ratio.
if (num_matches[i] == 1 &&
matches[cur_match_pos - 1].length() > kMaxZopfliLen) {
i += matches[cur_match_pos - 1].length() - 1;
queue.Clear();
}
}
ComputeShortestPathFromNodes(num_bytes, &nodes[0], path);
}
void ZopfliComputeShortestPath(size_t num_bytes,
size_t position,
const uint8_t* ringbuffer,
size_t ringbuffer_mask,
const size_t max_backward_limit,
const int* dist_cache,
Hashers::H10* hasher,
ZopfliNode* nodes,
std::vector<uint32_t>* path) {
nodes[0].length = 0;
nodes[0].cost = 0;
ZopfliCostModel* model = new ZopfliCostModel;
model->SetFromLiteralCosts(num_bytes, position,
ringbuffer, ringbuffer_mask);
StartPosQueue queue(3);
BackwardMatch matches[Hashers::H10::kMaxNumMatches];
for (size_t i = 0; i + 3 < num_bytes; i++) {
const size_t max_distance = std::min(position + i, max_backward_limit);
size_t num_matches = hasher->FindAllMatches(
ringbuffer, ringbuffer_mask, position + i, num_bytes - i, max_distance,
matches);
if (num_matches > 0 &&
matches[num_matches - 1].length() > kMaxZopfliLen) {
matches[0] = matches[num_matches - 1];
num_matches = 1;
}
UpdateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask,
max_backward_limit, dist_cache, num_matches, matches,
model, &queue, nodes);
if (num_matches == 1 && matches[0].length() > kMaxZopfliLen) {
for (size_t j = 1; j < matches[0].length() && i + 4 < num_bytes; ++j) {
++i;
if (matches[0].length() - j < 64 &&
num_bytes - i >= kMaxTreeCompLength) {
hasher->Store(ringbuffer, ringbuffer_mask, position + i);
}
}
queue.Clear();
}
}
delete model;
ComputeShortestPathFromNodes(num_bytes, nodes, path);
}
template<typename Hasher>
void CreateBackwardReferences(size_t num_bytes,
size_t position,
bool is_last,
const uint8_t* ringbuffer,
size_t ringbuffer_mask,
const int quality,
const int lgwin,
Hasher* hasher,
int* dist_cache,
size_t* last_insert_len,
Command* commands,
size_t* num_commands,
size_t* num_literals) {
// Set maximum distance, see section 9.1. of the spec.
const size_t max_backward_limit = (1 << lgwin) - 16;
// Choose which init method is faster.
// memset is about 100 times faster than hasher->InitForData().
const size_t kMaxBytesForPartialHashInit = Hasher::kHashMapSize >> 7;
if (position == 0 && is_last && num_bytes <= kMaxBytesForPartialHashInit) {
hasher->InitForData(ringbuffer, num_bytes);
} else {
hasher->Init();
}
if (num_bytes >= 3 && position >= 3) {
// Prepare the hashes for three last bytes of the last write.
// These could not be calculated before, since they require knowledge
// of both the previous and the current block.
hasher->Store(&ringbuffer[(position - 3) & ringbuffer_mask],
static_cast<uint32_t>(position - 3));
hasher->Store(&ringbuffer[(position - 2) & ringbuffer_mask],
static_cast<uint32_t>(position - 2));
hasher->Store(&ringbuffer[(position - 1) & ringbuffer_mask],
static_cast<uint32_t>(position - 1));
}
const Command * const orig_commands = commands;
size_t insert_length = *last_insert_len;
size_t i = position & ringbuffer_mask;
const size_t i_diff = position - i;
const size_t i_end = i + num_bytes;
// For speed up heuristics for random data.
const size_t random_heuristics_window_size = quality < 9 ? 64 : 512;
size_t apply_random_heuristics = i + random_heuristics_window_size;
// Minimum score to accept a backward reference.
const double kMinScore = 4.0;
while (i + Hasher::kHashTypeLength - 1 < i_end) {
size_t max_length = i_end - i;
size_t max_distance = std::min(i + i_diff, max_backward_limit);
size_t best_len = 0;
size_t best_len_code = 0;
size_t best_dist = 0;
double best_score = kMinScore;
bool match_found = hasher->FindLongestMatch(
ringbuffer, ringbuffer_mask,
dist_cache, static_cast<uint32_t>(i + i_diff), max_length, max_distance,
&best_len, &best_len_code, &best_dist, &best_score);
if (match_found) {
// Found a match. Let's look for something even better ahead.
int delayed_backward_references_in_row = 0;
for (;;) {
--max_length;
size_t best_len_2 =
quality < 5 ? std::min(best_len - 1, max_length) : 0;
size_t best_len_code_2 = 0;
size_t best_dist_2 = 0;
double best_score_2 = kMinScore;
max_distance = std::min(i + i_diff + 1, max_backward_limit);
match_found = hasher->FindLongestMatch(
ringbuffer, ringbuffer_mask,
dist_cache, static_cast<uint32_t>(i + i_diff + 1),
max_length, max_distance,
&best_len_2, &best_len_code_2, &best_dist_2, &best_score_2);
double cost_diff_lazy = 7.0;
if (match_found && best_score_2 >= best_score + cost_diff_lazy) {
// Ok, let's just write one byte for now and start a match from the
// next byte.
++i;
++insert_length;
best_len = best_len_2;
best_len_code = best_len_code_2;
best_dist = best_dist_2;
best_score = best_score_2;
if (++delayed_backward_references_in_row < 4) {
continue;
}
}
break;
}
apply_random_heuristics =
i + 2 * best_len + random_heuristics_window_size;
max_distance = std::min(i + i_diff, max_backward_limit);
// The first 16 codes are special shortcodes, and the minimum offset is 1.
size_t distance_code =
ComputeDistanceCode(best_dist, max_distance, quality, dist_cache);
if (best_dist <= max_distance && distance_code > 0) {
dist_cache[3] = dist_cache[2];
dist_cache[2] = dist_cache[1];
dist_cache[1] = dist_cache[0];
dist_cache[0] = static_cast<int>(best_dist);
}
Command cmd(insert_length, best_len, best_len_code, distance_code);
*commands++ = cmd;
*num_literals += insert_length;
insert_length = 0;
// Put the hash keys into the table, if there are enough
// bytes left.
for (size_t j = 2; j < best_len; ++j) {
hasher->Store(&ringbuffer[i + j],
static_cast<uint32_t>(i + i_diff + j));
}
i += best_len;
} else {
++insert_length;
++i;
// If we have not seen matches for a long time, we can skip some
// match lookups. Unsuccessful match lookups are very very expensive
// and this kind of a heuristic speeds up compression quite
// a lot.
if (i > apply_random_heuristics) {
// Going through uncompressible data, jump.
if (i > apply_random_heuristics + 4 * random_heuristics_window_size) {
// It is quite a long time since we saw a copy, so we assume
// that this data is not compressible, and store hashes less
// often. Hashes of non compressible data are less likely to
// turn out to be useful in the future, too, so we store less of
// them to not to flood out the hash table of good compressible
// data.
size_t i_jump = std::min(i + 16, i_end - 4);
for (; i < i_jump; i += 4) {
hasher->Store(ringbuffer + i, static_cast<uint32_t>(i + i_diff));
insert_length += 4;
}
} else {
size_t i_jump = std::min(i + 8, i_end - 3);
for (; i < i_jump; i += 2) {
hasher->Store(ringbuffer + i, static_cast<uint32_t>(i + i_diff));
insert_length += 2;
}
}
}
}
}
insert_length += i_end - i;
*last_insert_len = insert_length;
*num_commands += static_cast<size_t>(commands - orig_commands);
}
void CreateBackwardReferences(size_t num_bytes,
size_t position,
bool is_last,
const uint8_t* ringbuffer,
size_t ringbuffer_mask,
const int quality,
const int lgwin,
Hashers* hashers,
int hash_type,
int* dist_cache,
size_t* last_insert_len,
Command* commands,
size_t* num_commands,
size_t* num_literals) {
bool zopflify = quality > 9;
if (zopflify) {
Hashers::H10* hasher = hashers->hash_h10;
hasher->Init(lgwin, position, num_bytes, is_last);
hasher->StitchToPreviousBlock(num_bytes, position,
ringbuffer, ringbuffer_mask);
// Set maximum distance, see section 9.1. of the spec.
const size_t max_backward_limit = (1 << lgwin) - 16;
if (quality == 10) {
std::vector<ZopfliNode> nodes(num_bytes + 1);
std::vector<uint32_t> path;
ZopfliComputeShortestPath(num_bytes, position,
ringbuffer, ringbuffer_mask,
max_backward_limit, dist_cache, hasher,
&nodes[0], &path);
ZopfliCreateCommands(num_bytes, position, max_backward_limit, path,
&nodes[0], dist_cache, last_insert_len, commands,
num_literals);
*num_commands += path.size();
return;
}
std::vector<uint32_t> num_matches(num_bytes);
std::vector<BackwardMatch> matches(4 * num_bytes);
size_t cur_match_pos = 0;
for (size_t i = 0; i + 3 < num_bytes; ++i) {
size_t max_distance = std::min(position + i, max_backward_limit);
size_t max_length = num_bytes - i;
// Ensure that we have enough free slots.
if (matches.size() < cur_match_pos + Hashers::H10::kMaxNumMatches) {
matches.resize(cur_match_pos + Hashers::H10::kMaxNumMatches);
}
size_t num_found_matches = hasher->FindAllMatches(
ringbuffer, ringbuffer_mask, position + i, max_length, max_distance,
&matches[cur_match_pos]);
const size_t cur_match_end = cur_match_pos + num_found_matches;
for (size_t j = cur_match_pos; j + 1 < cur_match_end; ++j) {
assert(matches[j].length() < matches[j + 1].length());
assert(matches[j].distance > max_distance ||
matches[j].distance <= matches[j + 1].distance);
}
num_matches[i] = static_cast<uint32_t>(num_found_matches);
if (num_found_matches > 0) {
const size_t match_len = matches[cur_match_end - 1].length();
if (match_len > kMaxZopfliLen) {
matches[cur_match_pos++] = matches[cur_match_end - 1];
num_matches[i] = 1;
for (size_t j = 1; j < match_len; ++j) {
++i;
if (match_len - j < 64 && num_bytes - i >= kMaxTreeCompLength) {
hasher->Store(ringbuffer, ringbuffer_mask, position + i);
}
num_matches[i] = 0;
}
} else {
cur_match_pos = cur_match_end;
}
}
}
size_t orig_num_literals = *num_literals;
size_t orig_last_insert_len = *last_insert_len;
int orig_dist_cache[4] = {
dist_cache[0], dist_cache[1], dist_cache[2], dist_cache[3]
};
size_t orig_num_commands = *num_commands;
static const size_t kIterations = 2;
for (size_t i = 0; i < kIterations; i++) {
ZopfliCostModel model;
if (i == 0) {
model.SetFromLiteralCosts(num_bytes, position,
ringbuffer, ringbuffer_mask);
} else {
model.SetFromCommands(num_bytes, position,
ringbuffer, ringbuffer_mask,
commands, *num_commands - orig_num_commands,
orig_last_insert_len);
}
*num_commands = orig_num_commands;
*num_literals = orig_num_literals;
*last_insert_len = orig_last_insert_len;
memcpy(dist_cache, orig_dist_cache, 4 * sizeof(dist_cache[0]));
std::vector<ZopfliNode> nodes(num_bytes + 1);
std::vector<uint32_t> path;
ZopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask,
max_backward_limit, dist_cache, model, num_matches, matches,
&nodes[0], &path);
ZopfliCreateCommands(num_bytes, position, max_backward_limit, path,
&nodes[0], dist_cache, last_insert_len, commands,
num_literals);
*num_commands += path.size();
}
return;
}
switch (hash_type) {
case 2:
CreateBackwardReferences<Hashers::H2>(
num_bytes, position, is_last, ringbuffer, ringbuffer_mask,
quality, lgwin, hashers->hash_h2, dist_cache,
last_insert_len, commands, num_commands, num_literals);
break;
case 3:
CreateBackwardReferences<Hashers::H3>(
num_bytes, position, is_last, ringbuffer, ringbuffer_mask,
quality, lgwin, hashers->hash_h3, dist_cache,
last_insert_len, commands, num_commands, num_literals);
break;
case 4:
CreateBackwardReferences<Hashers::H4>(
num_bytes, position, is_last, ringbuffer, ringbuffer_mask,
quality, lgwin, hashers->hash_h4, dist_cache,
last_insert_len, commands, num_commands, num_literals);
break;
case 5:
CreateBackwardReferences<Hashers::H5>(
num_bytes, position, is_last, ringbuffer, ringbuffer_mask,
quality, lgwin, hashers->hash_h5, dist_cache,
last_insert_len, commands, num_commands, num_literals);
break;
case 6:
CreateBackwardReferences<Hashers::H6>(
num_bytes, position, is_last, ringbuffer, ringbuffer_mask,
quality, lgwin, hashers->hash_h6, dist_cache,
last_insert_len, commands, num_commands, num_literals);
break;
case 7:
CreateBackwardReferences<Hashers::H7>(
num_bytes, position, is_last, ringbuffer, ringbuffer_mask,
quality, lgwin, hashers->hash_h7, dist_cache,
last_insert_len, commands, num_commands, num_literals);
break;
case 8:
CreateBackwardReferences<Hashers::H8>(
num_bytes, position, is_last, ringbuffer, ringbuffer_mask,
quality, lgwin, hashers->hash_h8, dist_cache,
last_insert_len, commands, num_commands, num_literals);
break;
case 9:
CreateBackwardReferences<Hashers::H9>(
num_bytes, position, is_last, ringbuffer, ringbuffer_mask,
quality, lgwin, hashers->hash_h9, dist_cache,
last_insert_len, commands, num_commands, num_literals);
break;
default:
break;
}
}
} // namespace brotli

125
modules/brotli/enc/backward_references.h
View File

@@ -4,113 +4,36 @@
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
// Function to find backward reference copies.
/* Function to find backward reference copies. */
#ifndef BROTLI_ENC_BACKWARD_REFERENCES_H_
#define BROTLI_ENC_BACKWARD_REFERENCES_H_
#include <vector>
#include "./hash.h"
#include "../common/constants.h"
#include "../common/dictionary.h"
#include <brotli/types.h>
#include "./command.h"
#include "./types.h"
#include "./hash.h"
#include "./port.h"
#include "./quality.h"
namespace brotli {
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
// "commands" points to the next output command to write to, "*num_commands" is
// initially the total amount of commands output by previous
// CreateBackwardReferences calls, and must be incremented by the amount written
// by this call.
void CreateBackwardReferences(size_t num_bytes,
size_t position,
bool is_last,
const uint8_t* ringbuffer,
size_t ringbuffer_mask,
const int quality,
const int lgwin,
Hashers* hashers,
int hash_type,
int* dist_cache,
size_t* last_insert_len,
Command* commands,
size_t* num_commands,
size_t* num_literals);
/* "commands" points to the next output command to write to, "*num_commands" is
initially the total amount of commands output by previous
CreateBackwardReferences calls, and must be incremented by the amount written
by this call. */
BROTLI_INTERNAL void BrotliCreateBackwardReferences(
const BrotliDictionary* dictionary, size_t num_bytes, size_t position,
const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
size_t* last_insert_len, Command* commands, size_t* num_commands,
size_t* num_literals);
static const float kInfinity = std::numeric_limits<float>::infinity();
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
struct ZopfliNode {
ZopfliNode(void) : length(1),
distance(0),
insert_length(0),
cost(kInfinity) {}
inline uint32_t copy_length() const {
return length & 0xffffff;
}
inline uint32_t length_code() const {
const uint32_t modifier = length >> 24;
return copy_length() + 9u - modifier;
}
inline uint32_t copy_distance() const {
return distance & 0x1ffffff;
}
inline uint32_t distance_code() const {
const uint32_t short_code = distance >> 25;
return short_code == 0 ? copy_distance() + 15 : short_code - 1;
}
inline uint32_t command_length() const {
return copy_length() + insert_length;
}
// best length to get up to this byte (not including this byte itself)
// highest 8 bit is used to reconstruct the length code
uint32_t length;
// distance associated with the length
// highest 7 bit contains distance short code + 1 (or zero if no short code)
uint32_t distance;
// number of literal inserts before this copy
uint32_t insert_length;
// smallest cost to get to this byte from the beginning, as found so far
float cost;
};
// Computes the shortest path of commands from position to at most
// position + num_bytes.
//
// On return, path->size() is the number of commands found and path[i] is the
// length of the ith command (copy length plus insert length).
// Note that the sum of the lengths of all commands can be less than num_bytes.
//
// On return, the nodes[0..num_bytes] array will have the following
// "ZopfliNode array invariant":
// For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then
// (1) nodes[i].copy_length() >= 2
// (2) nodes[i].command_length() <= i and
// (3) nodes[i - nodes[i].command_length()].cost < kInfinity
void ZopfliComputeShortestPath(size_t num_bytes,
size_t position,
const uint8_t* ringbuffer,
size_t ringbuffer_mask,
const size_t max_backward_limit,
const int* dist_cache,
Hashers::H10* hasher,
ZopfliNode* nodes,
std::vector<uint32_t>* path);
void ZopfliCreateCommands(const size_t num_bytes,
const size_t block_start,
const size_t max_backward_limit,
const std::vector<uint32_t>& path,
const ZopfliNode* nodes,
int* dist_cache,
size_t* last_insert_len,
Command* commands,
size_t* num_literals);
} // namespace brotli
#endif // BROTLI_ENC_BACKWARD_REFERENCES_H_
#endif /* BROTLI_ENC_BACKWARD_REFERENCES_H_ */

790
modules/brotli/enc/backward_references_hq.c Executable file
View File

@@ -0,0 +1,790 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Function to find backward reference copies. */
#include "./backward_references_hq.h"
#include <string.h> /* memcpy, memset */
#include "../common/constants.h"
#include <brotli/types.h>
#include "./command.h"
#include "./fast_log.h"
#include "./find_match_length.h"
#include "./literal_cost.h"
#include "./memory.h"
#include "./port.h"
#include "./prefix.h"
#include "./quality.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
static const float kInfinity = 1.7e38f; /* ~= 2 ^ 127 */
static const uint32_t kDistanceCacheIndex[] = {
0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1,
};
static const int kDistanceCacheOffset[] = {
0, 0, 0, 0, -1, 1, -2, 2, -3, 3, -1, 1, -2, 2, -3, 3
};
void BrotliInitZopfliNodes(ZopfliNode* array, size_t length) {
ZopfliNode stub;
size_t i;
stub.length = 1;
stub.distance = 0;
stub.insert_length = 0;
stub.u.cost = kInfinity;
for (i = 0; i < length; ++i) array[i] = stub;
}
static BROTLI_INLINE uint32_t ZopfliNodeCopyLength(const ZopfliNode* self) {
return self->length & 0xffffff;
}
static BROTLI_INLINE uint32_t ZopfliNodeLengthCode(const ZopfliNode* self) {
const uint32_t modifier = self->length >> 24;
return ZopfliNodeCopyLength(self) + 9u - modifier;
}
static BROTLI_INLINE uint32_t ZopfliNodeCopyDistance(const ZopfliNode* self) {
return self->distance & 0x1ffffff;
}
static BROTLI_INLINE uint32_t ZopfliNodeDistanceCode(const ZopfliNode* self) {
const uint32_t short_code = self->distance >> 25;
return short_code == 0 ?
ZopfliNodeCopyDistance(self) + BROTLI_NUM_DISTANCE_SHORT_CODES - 1 :
short_code - 1;
}
static BROTLI_INLINE uint32_t ZopfliNodeCommandLength(const ZopfliNode* self) {
return ZopfliNodeCopyLength(self) + self->insert_length;
}
/* Histogram based cost model for zopflification. */
typedef struct ZopfliCostModel {
/* The insert and copy length symbols. */
float cost_cmd_[BROTLI_NUM_COMMAND_SYMBOLS];
float cost_dist_[BROTLI_NUM_DISTANCE_SYMBOLS];
/* Cumulative costs of literals per position in the stream. */
float* literal_costs_;
float min_cost_cmd_;
size_t num_bytes_;
} ZopfliCostModel;
static void InitZopfliCostModel(
MemoryManager* m, ZopfliCostModel* self, size_t num_bytes) {
self->num_bytes_ = num_bytes;
self->literal_costs_ = BROTLI_ALLOC(m, float, num_bytes + 2);
if (BROTLI_IS_OOM(m)) return;
}
static void CleanupZopfliCostModel(MemoryManager* m, ZopfliCostModel* self) {
BROTLI_FREE(m, self->literal_costs_);
}
static void SetCost(const uint32_t* histogram, size_t histogram_size,
float* cost) {
size_t sum = 0;
float log2sum;
size_t i;
for (i = 0; i < histogram_size; i++) {
sum += histogram[i];
}
log2sum = (float)FastLog2(sum);
for (i = 0; i < histogram_size; i++) {
if (histogram[i] == 0) {
cost[i] = log2sum + 2;
continue;
}
/* Shannon bits for this symbol. */
cost[i] = log2sum - (float)FastLog2(histogram[i]);
/* Cannot be coded with less than 1 bit */
if (cost[i] < 1) cost[i] = 1;
}
}
static void ZopfliCostModelSetFromCommands(ZopfliCostModel* self,
size_t position,
const uint8_t* ringbuffer,
size_t ringbuffer_mask,
const Command* commands,
size_t num_commands,
size_t last_insert_len) {
uint32_t histogram_literal[BROTLI_NUM_LITERAL_SYMBOLS];
uint32_t histogram_cmd[BROTLI_NUM_COMMAND_SYMBOLS];
uint32_t histogram_dist[BROTLI_NUM_DISTANCE_SYMBOLS];
float cost_literal[BROTLI_NUM_LITERAL_SYMBOLS];
size_t pos = position - last_insert_len;
float min_cost_cmd = kInfinity;
size_t i;
float* cost_cmd = self->cost_cmd_;
memset(histogram_literal, 0, sizeof(histogram_literal));
memset(histogram_cmd, 0, sizeof(histogram_cmd));
memset(histogram_dist, 0, sizeof(histogram_dist));
for (i = 0; i < num_commands; i++) {
size_t inslength = commands[i].insert_len_;
size_t copylength = CommandCopyLen(&commands[i]);
size_t distcode = commands[i].dist_prefix_;
size_t cmdcode = commands[i].cmd_prefix_;
size_t j;
histogram_cmd[cmdcode]++;
if (cmdcode >= 128) histogram_dist[distcode]++;
for (j = 0; j < inslength; j++) {
histogram_literal[ringbuffer[(pos + j) & ringbuffer_mask]]++;
}
pos += inslength + copylength;
}
SetCost(histogram_literal, BROTLI_NUM_LITERAL_SYMBOLS, cost_literal);
SetCost(histogram_cmd, BROTLI_NUM_COMMAND_SYMBOLS, cost_cmd);
SetCost(histogram_dist, BROTLI_NUM_DISTANCE_SYMBOLS, self->cost_dist_);
for (i = 0; i < BROTLI_NUM_COMMAND_SYMBOLS; ++i) {
min_cost_cmd = BROTLI_MIN(float, min_cost_cmd, cost_cmd[i]);
}
self->min_cost_cmd_ = min_cost_cmd;
{
float* literal_costs = self->literal_costs_;
size_t num_bytes = self->num_bytes_;
literal_costs[0] = 0.0;
for (i = 0; i < num_bytes; ++i) {
literal_costs[i + 1] = literal_costs[i] +
cost_literal[ringbuffer[(position + i) & ringbuffer_mask]];
}
}
}
static void ZopfliCostModelSetFromLiteralCosts(ZopfliCostModel* self,
size_t position,
const uint8_t* ringbuffer,
size_t ringbuffer_mask) {
float* literal_costs = self->literal_costs_;
float* cost_dist = self->cost_dist_;
float* cost_cmd = self->cost_cmd_;
size_t num_bytes = self->num_bytes_;
size_t i;
BrotliEstimateBitCostsForLiterals(position, num_bytes, ringbuffer_mask,
ringbuffer, &literal_costs[1]);
literal_costs[0] = 0.0;
for (i = 0; i < num_bytes; ++i) {
literal_costs[i + 1] += literal_costs[i];
}
for (i = 0; i < BROTLI_NUM_COMMAND_SYMBOLS; ++i) {
cost_cmd[i] = (float)FastLog2(11 + (uint32_t)i);
}
for (i = 0; i < BROTLI_NUM_DISTANCE_SYMBOLS; ++i) {
cost_dist[i] = (float)FastLog2(20 + (uint32_t)i);
}
self->min_cost_cmd_ = (float)FastLog2(11);
}
static BROTLI_INLINE float ZopfliCostModelGetCommandCost(
const ZopfliCostModel* self, uint16_t cmdcode) {
return self->cost_cmd_[cmdcode];
}
static BROTLI_INLINE float ZopfliCostModelGetDistanceCost(
const ZopfliCostModel* self, size_t distcode) {
return self->cost_dist_[distcode];
}
static BROTLI_INLINE float ZopfliCostModelGetLiteralCosts(
const ZopfliCostModel* self, size_t from, size_t to) {
return self->literal_costs_[to] - self->literal_costs_[from];
}
static BROTLI_INLINE float ZopfliCostModelGetMinCostCmd(
const ZopfliCostModel* self) {
return self->min_cost_cmd_;
}
/* REQUIRES: len >= 2, start_pos <= pos */
/* REQUIRES: cost < kInfinity, nodes[start_pos].cost < kInfinity */
/* Maintains the "ZopfliNode array invariant". */
static BROTLI_INLINE void UpdateZopfliNode(ZopfliNode* nodes, size_t pos,
size_t start_pos, size_t len, size_t len_code, size_t dist,
size_t short_code, float cost) {
ZopfliNode* next = &nodes[pos + len];
next->length = (uint32_t)(len | ((len + 9u - len_code) << 24));
next->distance = (uint32_t)(dist | (short_code << 25));
next->insert_length = (uint32_t)(pos - start_pos);
next->u.cost = cost;
}
typedef struct PosData {
size_t pos;
int distance_cache[4];
float costdiff;
float cost;
} PosData;
/* Maintains the smallest 8 cost difference together with their positions */
typedef struct StartPosQueue {
PosData q_[8];
size_t idx_;
} StartPosQueue;
static BROTLI_INLINE void InitStartPosQueue(StartPosQueue* self) {
self->idx_ = 0;
}
static size_t StartPosQueueSize(const StartPosQueue* self) {
return BROTLI_MIN(size_t, self->idx_, 8);
}
static void StartPosQueuePush(StartPosQueue* self, const PosData* posdata) {
size_t offset = ~(self->idx_++) & 7;
size_t len = StartPosQueueSize(self);
size_t i;
PosData* q = self->q_;
q[offset] = *posdata;
/* Restore the sorted order. In the list of |len| items at most |len - 1|
adjacent element comparisons / swaps are required. */
for (i = 1; i < len; ++i) {
if (q[offset & 7].costdiff > q[(offset + 1) & 7].costdiff) {
BROTLI_SWAP(PosData, q, offset & 7, (offset + 1) & 7);
}
++offset;
}
}
static const PosData* StartPosQueueAt(const StartPosQueue* self, size_t k) {
return &self->q_[(k - self->idx_) & 7];
}
/* Returns the minimum possible copy length that can improve the cost of any */
/* future position. */
static size_t ComputeMinimumCopyLength(const float start_cost,
const ZopfliNode* nodes,
const size_t num_bytes,
const size_t pos) {
/* Compute the minimum possible cost of reaching any future position. */
float min_cost = start_cost;
size_t len = 2;
size_t next_len_bucket = 4;
size_t next_len_offset = 10;
while (pos + len <= num_bytes && nodes[pos + len].u.cost <= min_cost) {
/* We already reached (pos + len) with no more cost than the minimum
possible cost of reaching anything from this pos, so there is no point in
looking for lengths <= len. */
++len;
if (len == next_len_offset) {
/* We reached the next copy length code bucket, so we add one more
extra bit to the minimum cost. */
min_cost += 1.0f;
next_len_offset += next_len_bucket;
next_len_bucket *= 2;
}
}
return len;
}
/* REQUIRES: nodes[pos].cost < kInfinity
REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */
static uint32_t ComputeDistanceShortcut(const size_t block_start,
const size_t pos,
const size_t max_backward,
const ZopfliNode* nodes) {
const size_t clen = ZopfliNodeCopyLength(&nodes[pos]);
const size_t ilen = nodes[pos].insert_length;
const size_t dist = ZopfliNodeCopyDistance(&nodes[pos]);
/* Since |block_start + pos| is the end position of the command, the copy part
starts from |block_start + pos - clen|. Distances that are greater than
this or greater than |max_backward| are static dictionary references, and
do not update the last distances. Also distance code 0 (last distance)
does not update the last distances. */
if (pos == 0) {
return 0;
} else if (dist + clen <= block_start + pos &&
dist <= max_backward &&
ZopfliNodeDistanceCode(&nodes[pos]) > 0) {
return (uint32_t)pos;
} else {
return nodes[pos - clen - ilen].u.shortcut;
}
}
/* Fills in dist_cache[0..3] with the last four distances (as defined by
Section 4. of the Spec) that would be used at (block_start + pos) if we
used the shortest path of commands from block_start, computed from
nodes[0..pos]. The last four distances at block_start are in
starting_dist_cache[0..3].
REQUIRES: nodes[pos].cost < kInfinity
REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */
static void ComputeDistanceCache(const size_t pos,
const int* starting_dist_cache,
const ZopfliNode* nodes,
int* dist_cache) {
int idx = 0;
size_t p = nodes[pos].u.shortcut;
while (idx < 4 && p > 0) {
const size_t ilen = nodes[p].insert_length;
const size_t clen = ZopfliNodeCopyLength(&nodes[p]);
const size_t dist = ZopfliNodeCopyDistance(&nodes[p]);
dist_cache[idx++] = (int)dist;
/* Because of prerequisite, p >= clen + ilen >= 2. */
p = nodes[p - clen - ilen].u.shortcut;
}
for (; idx < 4; ++idx) {
dist_cache[idx] = *starting_dist_cache++;
}
}
/* Maintains "ZopfliNode array invariant" and pushes node to the queue, if it
is eligible. */
static void EvaluateNode(
const size_t block_start, const size_t pos, const size_t max_backward_limit,
const int* starting_dist_cache, const ZopfliCostModel* model,
StartPosQueue* queue, ZopfliNode* nodes) {
/* Save cost, because ComputeDistanceCache invalidates it. */
float node_cost = nodes[pos].u.cost;
nodes[pos].u.shortcut = ComputeDistanceShortcut(
block_start, pos, max_backward_limit, nodes);
if (node_cost <= ZopfliCostModelGetLiteralCosts(model, 0, pos)) {
PosData posdata;
posdata.pos = pos;
posdata.cost = node_cost;
posdata.costdiff = node_cost -
ZopfliCostModelGetLiteralCosts(model, 0, pos);
ComputeDistanceCache(
pos, starting_dist_cache, nodes, posdata.distance_cache);
StartPosQueuePush(queue, &posdata);
}
}
/* Returns longest copy length. */
static size_t UpdateNodes(
const size_t num_bytes, const size_t block_start, const size_t pos,
const uint8_t* ringbuffer, const size_t ringbuffer_mask,
const BrotliEncoderParams* params, const size_t max_backward_limit,
const int* starting_dist_cache, const size_t num_matches,
const BackwardMatch* matches, const ZopfliCostModel* model,
StartPosQueue* queue, ZopfliNode* nodes) {
const size_t cur_ix = block_start + pos;
const size_t cur_ix_masked = cur_ix & ringbuffer_mask;
const size_t max_distance = BROTLI_MIN(size_t, cur_ix, max_backward_limit);
const size_t max_len = num_bytes - pos;
const size_t max_zopfli_len = MaxZopfliLen(params);
const size_t max_iters = MaxZopfliCandidates(params);
size_t min_len;
size_t result = 0;
size_t k;
EvaluateNode(block_start, pos, max_backward_limit, starting_dist_cache, model,
queue, nodes);
{
const PosData* posdata = StartPosQueueAt(queue, 0);
float min_cost = (posdata->cost + ZopfliCostModelGetMinCostCmd(model) +
ZopfliCostModelGetLiteralCosts(model, posdata->pos, pos));
min_len = ComputeMinimumCopyLength(min_cost, nodes, num_bytes, pos);
}
/* Go over the command starting positions in order of increasing cost
difference. */
for (k = 0; k < max_iters && k < StartPosQueueSize(queue); ++k) {
const PosData* posdata = StartPosQueueAt(queue, k);
const size_t start = posdata->pos;
const uint16_t inscode = GetInsertLengthCode(pos - start);
const float start_costdiff = posdata->costdiff;
const float base_cost = start_costdiff + (float)GetInsertExtra(inscode) +
ZopfliCostModelGetLiteralCosts(model, 0, pos);
/* Look for last distance matches using the distance cache from this
starting position. */
size_t best_len = min_len - 1;
size_t j = 0;
for (; j < BROTLI_NUM_DISTANCE_SHORT_CODES && best_len < max_len; ++j) {
const size_t idx = kDistanceCacheIndex[j];
const size_t backward =
(size_t)(posdata->distance_cache[idx] + kDistanceCacheOffset[j]);
size_t prev_ix = cur_ix - backward;
if (prev_ix >= cur_ix) {
continue;
}
if (BROTLI_PREDICT_FALSE(backward > max_distance)) {
continue;
}
prev_ix &= ringbuffer_mask;
if (cur_ix_masked + best_len > ringbuffer_mask ||
prev_ix + best_len > ringbuffer_mask ||
ringbuffer[cur_ix_masked + best_len] !=
ringbuffer[prev_ix + best_len]) {
continue;
}
{
const size_t len =
FindMatchLengthWithLimit(&ringbuffer[prev_ix],
&ringbuffer[cur_ix_masked],
max_len);
const float dist_cost = base_cost +
ZopfliCostModelGetDistanceCost(model, j);
size_t l;
for (l = best_len + 1; l <= len; ++l) {
const uint16_t copycode = GetCopyLengthCode(l);
const uint16_t cmdcode =
CombineLengthCodes(inscode, copycode, j == 0);
const float cost = (cmdcode < 128 ? base_cost : dist_cost) +
(float)GetCopyExtra(copycode) +
ZopfliCostModelGetCommandCost(model, cmdcode);
if (cost < nodes[pos + l].u.cost) {
UpdateZopfliNode(nodes, pos, start, l, l, backward, j + 1, cost);
result = BROTLI_MAX(size_t, result, l);
}
best_len = l;
}
}
}
/* At higher iterations look only for new last distance matches, since
looking only for new command start positions with the same distances
does not help much. */
if (k >= 2) continue;
{
/* Loop through all possible copy lengths at this position. */
size_t len = min_len;
for (j = 0; j < num_matches; ++j) {
BackwardMatch match = matches[j];
size_t dist = match.distance;
BROTLI_BOOL is_dictionary_match = TO_BROTLI_BOOL(dist > max_distance);
/* We already tried all possible last distance matches, so we can use
normal distance code here. */
size_t dist_code = dist + BROTLI_NUM_DISTANCE_SHORT_CODES - 1;
uint16_t dist_symbol;
uint32_t distextra;
uint32_t distnumextra;
float dist_cost;
size_t max_match_len;
PrefixEncodeCopyDistance(dist_code, 0, 0, &dist_symbol, &distextra);
distnumextra = distextra >> 24;
dist_cost = base_cost + (float)distnumextra +
ZopfliCostModelGetDistanceCost(model, dist_symbol);
/* Try all copy lengths up until the maximum copy length corresponding
to this distance. If the distance refers to the static dictionary, or
the maximum length is long enough, try only one maximum length. */
max_match_len = BackwardMatchLength(&match);
if (len < max_match_len &&
(is_dictionary_match || max_match_len > max_zopfli_len)) {
len = max_match_len;
}
for (; len <= max_match_len; ++len) {
const size_t len_code =
is_dictionary_match ? BackwardMatchLengthCode(&match) : len;
const uint16_t copycode = GetCopyLengthCode(len_code);
const uint16_t cmdcode = CombineLengthCodes(inscode, copycode, 0);
const float cost = dist_cost + (float)GetCopyExtra(copycode) +
ZopfliCostModelGetCommandCost(model, cmdcode);
if (cost < nodes[pos + len].u.cost) {
UpdateZopfliNode(nodes, pos, start, len, len_code, dist, 0, cost);
result = BROTLI_MAX(size_t, result, len);
}
}
}
}
}
return result;
}
static size_t ComputeShortestPathFromNodes(size_t num_bytes,
ZopfliNode* nodes) {
size_t index = num_bytes;
size_t num_commands = 0;
while (nodes[index].insert_length == 0 && nodes[index].length == 1) --index;
nodes[index].u.next = BROTLI_UINT32_MAX;
while (index != 0) {
size_t len = ZopfliNodeCommandLength(&nodes[index]);
index -= len;
nodes[index].u.next = (uint32_t)len;
num_commands++;
}
return num_commands;
}
/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
void BrotliZopfliCreateCommands(const size_t num_bytes,
const size_t block_start,
const size_t max_backward_limit,
const ZopfliNode* nodes,
int* dist_cache,
size_t* last_insert_len,
Command* commands,
size_t* num_literals) {
size_t pos = 0;
uint32_t offset = nodes[0].u.next;
size_t i;
for (i = 0; offset != BROTLI_UINT32_MAX; i++) {
const ZopfliNode* next = &nodes[pos + offset];
size_t copy_length = ZopfliNodeCopyLength(next);
size_t insert_length = next->insert_length;
pos += insert_length;
offset = next->u.next;
if (i == 0) {
insert_length += *last_insert_len;
*last_insert_len = 0;
}
{
size_t distance = ZopfliNodeCopyDistance(next);
size_t len_code = ZopfliNodeLengthCode(next);
size_t max_distance =
BROTLI_MIN(size_t, block_start + pos, max_backward_limit);
BROTLI_BOOL is_dictionary = TO_BROTLI_BOOL(distance > max_distance);
size_t dist_code = ZopfliNodeDistanceCode(next);
InitCommand(
&commands[i], insert_length, copy_length, len_code, dist_code);
if (!is_dictionary && dist_code > 0) {
dist_cache[3] = dist_cache[2];
dist_cache[2] = dist_cache[1];
dist_cache[1] = dist_cache[0];
dist_cache[0] = (int)distance;
}
}
*num_literals += insert_length;
pos += copy_length;
}
*last_insert_len += num_bytes - pos;
}
static size_t ZopfliIterate(size_t num_bytes,
size_t position,
const uint8_t* ringbuffer,
size_t ringbuffer_mask,
const BrotliEncoderParams* params,
const size_t max_backward_limit,
const int* dist_cache,
const ZopfliCostModel* model,
const uint32_t* num_matches,
const BackwardMatch* matches,
ZopfliNode* nodes) {
const size_t max_zopfli_len = MaxZopfliLen(params);
StartPosQueue queue;
size_t cur_match_pos = 0;
size_t i;
nodes[0].length = 0;
nodes[0].u.cost = 0;
InitStartPosQueue(&queue);
for (i = 0; i + 3 < num_bytes; i++) {
size_t skip = UpdateNodes(num_bytes, position, i, ringbuffer,
ringbuffer_mask, params, max_backward_limit, dist_cache,
num_matches[i], &matches[cur_match_pos], model, &queue, nodes);
if (skip < BROTLI_LONG_COPY_QUICK_STEP) skip = 0;
cur_match_pos += num_matches[i];
if (num_matches[i] == 1 &&
BackwardMatchLength(&matches[cur_match_pos - 1]) > max_zopfli_len) {
skip = BROTLI_MAX(size_t,
BackwardMatchLength(&matches[cur_match_pos - 1]), skip);
}
if (skip > 1) {
skip--;
while (skip) {
i++;
if (i + 3 >= num_bytes) break;
EvaluateNode(
position, i, max_backward_limit, dist_cache, model, &queue, nodes);
cur_match_pos += num_matches[i];
skip--;
}
}
}
return ComputeShortestPathFromNodes(num_bytes, nodes);
}
/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
const BrotliDictionary* dictionary,
size_t num_bytes,
size_t position,
const uint8_t* ringbuffer,
size_t ringbuffer_mask,
const BrotliEncoderParams* params,
const size_t max_backward_limit,
const int* dist_cache,
HasherHandle hasher,
ZopfliNode* nodes) {
const size_t max_zopfli_len = MaxZopfliLen(params);
ZopfliCostModel model;
StartPosQueue queue;
BackwardMatch matches[MAX_NUM_MATCHES_H10];
const size_t store_end = num_bytes >= StoreLookaheadH10() ?
position + num_bytes - StoreLookaheadH10() + 1 : position;
size_t i;
nodes[0].length = 0;
nodes[0].u.cost = 0;
InitZopfliCostModel(m, &model, num_bytes);
if (BROTLI_IS_OOM(m)) return 0;
ZopfliCostModelSetFromLiteralCosts(
&model, position, ringbuffer, ringbuffer_mask);
InitStartPosQueue(&queue);
for (i = 0; i + HashTypeLengthH10() - 1 < num_bytes; i++) {
const size_t pos = position + i;
const size_t max_distance = BROTLI_MIN(size_t, pos, max_backward_limit);
size_t num_matches = FindAllMatchesH10(hasher, dictionary, ringbuffer,
ringbuffer_mask, pos, num_bytes - i, max_distance, params, matches);
size_t skip;
if (num_matches > 0 &&
BackwardMatchLength(&matches[num_matches - 1]) > max_zopfli_len) {
matches[0] = matches[num_matches - 1];
num_matches = 1;
}
skip = UpdateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask,
params, max_backward_limit, dist_cache, num_matches, matches, &model,
&queue, nodes);
if (skip < BROTLI_LONG_COPY_QUICK_STEP) skip = 0;
if (num_matches == 1 && BackwardMatchLength(&matches[0]) > max_zopfli_len) {
skip = BROTLI_MAX(size_t, BackwardMatchLength(&matches[0]), skip);
}
if (skip > 1) {
/* Add the tail of the copy to the hasher. */
StoreRangeH10(hasher, ringbuffer, ringbuffer_mask, pos + 1, BROTLI_MIN(
size_t, pos + skip, store_end));
skip--;
while (skip) {
i++;
if (i + HashTypeLengthH10() - 1 >= num_bytes) break;
EvaluateNode(
position, i, max_backward_limit, dist_cache, &model, &queue, nodes);
skip--;
}
}
}
CleanupZopfliCostModel(m, &model);
return ComputeShortestPathFromNodes(num_bytes, nodes);
}
void BrotliCreateZopfliBackwardReferences(
MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
size_t* last_insert_len, Command* commands, size_t* num_commands,
size_t* num_literals) {
const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
ZopfliNode* nodes;
nodes = BROTLI_ALLOC(m, ZopfliNode, num_bytes + 1);
if (BROTLI_IS_OOM(m)) return;
BrotliInitZopfliNodes(nodes, num_bytes + 1);
*num_commands += BrotliZopfliComputeShortestPath(m, dictionary, num_bytes,
position, ringbuffer, ringbuffer_mask, params, max_backward_limit,
dist_cache, hasher, nodes);
if (BROTLI_IS_OOM(m)) return;
BrotliZopfliCreateCommands(num_bytes, position, max_backward_limit, nodes,
dist_cache, last_insert_len, commands, num_literals);
BROTLI_FREE(m, nodes);
}
void BrotliCreateHqZopfliBackwardReferences(
MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
size_t* last_insert_len, Command* commands, size_t* num_commands,
size_t* num_literals) {
const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
uint32_t* num_matches = BROTLI_ALLOC(m, uint32_t, num_bytes);
size_t matches_size = 4 * num_bytes;
const size_t store_end = num_bytes >= StoreLookaheadH10() ?
position + num_bytes - StoreLookaheadH10() + 1 : position;
size_t cur_match_pos = 0;
size_t i;
size_t orig_num_literals;
size_t orig_last_insert_len;
int orig_dist_cache[4];
size_t orig_num_commands;
ZopfliCostModel model;
ZopfliNode* nodes;
BackwardMatch* matches = BROTLI_ALLOC(m, BackwardMatch, matches_size);
if (BROTLI_IS_OOM(m)) return;
for (i = 0; i + HashTypeLengthH10() - 1 < num_bytes; ++i) {
const size_t pos = position + i;
size_t max_distance = BROTLI_MIN(size_t, pos, max_backward_limit);
size_t max_length = num_bytes - i;
size_t num_found_matches;
size_t cur_match_end;
size_t j;
/* Ensure that we have enough free slots. */
BROTLI_ENSURE_CAPACITY(m, BackwardMatch, matches, matches_size,
cur_match_pos + MAX_NUM_MATCHES_H10);
if (BROTLI_IS_OOM(m)) return;
num_found_matches = FindAllMatchesH10(hasher, dictionary, ringbuffer,
ringbuffer_mask, pos, max_length, max_distance, params,
&matches[cur_match_pos]);
cur_match_end = cur_match_pos + num_found_matches;
for (j = cur_match_pos; j + 1 < cur_match_end; ++j) {
assert(BackwardMatchLength(&matches[j]) <
BackwardMatchLength(&matches[j + 1]));
assert(matches[j].distance > max_distance ||
matches[j].distance <= matches[j + 1].distance);
}
num_matches[i] = (uint32_t)num_found_matches;
if (num_found_matches > 0) {
const size_t match_len = BackwardMatchLength(&matches[cur_match_end - 1]);
if (match_len > MAX_ZOPFLI_LEN_QUALITY_11) {
const size_t skip = match_len - 1;
matches[cur_match_pos++] = matches[cur_match_end - 1];
num_matches[i] = 1;
/* Add the tail of the copy to the hasher. */
StoreRangeH10(hasher, ringbuffer, ringbuffer_mask, pos + 1,
BROTLI_MIN(size_t, pos + match_len, store_end));
memset(&num_matches[i + 1], 0, skip * sizeof(num_matches[0]));
i += skip;
} else {
cur_match_pos = cur_match_end;
}
}
}
orig_num_literals = *num_literals;
orig_last_insert_len = *last_insert_len;
memcpy(orig_dist_cache, dist_cache, 4 * sizeof(dist_cache[0]));
orig_num_commands = *num_commands;
nodes = BROTLI_ALLOC(m, ZopfliNode, num_bytes + 1);
if (BROTLI_IS_OOM(m)) return;
InitZopfliCostModel(m, &model, num_bytes);
if (BROTLI_IS_OOM(m)) return;
for (i = 0; i < 2; i++) {
BrotliInitZopfliNodes(nodes, num_bytes + 1);
if (i == 0) {
ZopfliCostModelSetFromLiteralCosts(
&model, position, ringbuffer, ringbuffer_mask);
} else {
ZopfliCostModelSetFromCommands(&model, position, ringbuffer,
ringbuffer_mask, commands, *num_commands - orig_num_commands,
orig_last_insert_len);
}
*num_commands = orig_num_commands;
*num_literals = orig_num_literals;
*last_insert_len = orig_last_insert_len;
memcpy(dist_cache, orig_dist_cache, 4 * sizeof(dist_cache[0]));
*num_commands += ZopfliIterate(num_bytes, position, ringbuffer,
ringbuffer_mask, params, max_backward_limit, dist_cache,
&model, num_matches, matches, nodes);
BrotliZopfliCreateCommands(num_bytes, position, max_backward_limit,
nodes, dist_cache, last_insert_len, commands, num_literals);
}
CleanupZopfliCostModel(m, &model);
BROTLI_FREE(m, nodes);
BROTLI_FREE(m, matches);
BROTLI_FREE(m, num_matches);
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

99
modules/brotli/enc/backward_references_hq.h Executable file
View File

@@ -0,0 +1,99 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Function to find backward reference copies. */
#ifndef BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_
#define BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_
#include "../common/constants.h"
#include "../common/dictionary.h"
#include <brotli/types.h>
#include "./command.h"
#include "./hash.h"
#include "./memory.h"
#include "./port.h"
#include "./quality.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
BROTLI_INTERNAL void BrotliCreateZopfliBackwardReferences(
MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
size_t* last_insert_len, Command* commands, size_t* num_commands,
size_t* num_literals);
BROTLI_INTERNAL void BrotliCreateHqZopfliBackwardReferences(
MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
size_t* last_insert_len, Command* commands, size_t* num_commands,
size_t* num_literals);
typedef struct ZopfliNode {
/* best length to get up to this byte (not including this byte itself)
highest 8 bit is used to reconstruct the length code */
uint32_t length;
/* distance associated with the length
highest 7 bit contains distance short code + 1 (or zero if no short code)
*/
uint32_t distance;
/* number of literal inserts before this copy */
uint32_t insert_length;
/* This union holds information used by dynamic-programming. During forward
pass |cost| it used to store the goal function. When node is processed its
|cost| is invalidated in favor of |shortcut|. On path back-tracing pass
|next| is assigned the offset to next node on the path. */
union {
/* Smallest cost to get to this byte from the beginning, as found so far. */
float cost;
/* Offset to the next node on the path. Equals to command_length() of the
next node on the path. For last node equals to BROTLI_UINT32_MAX */
uint32_t next;
/* Node position that provides next distance for distance cache. */
uint32_t shortcut;
} u;
} ZopfliNode;
BROTLI_INTERNAL void BrotliInitZopfliNodes(ZopfliNode* array, size_t length);
/* Computes the shortest path of commands from position to at most
position + num_bytes.
On return, path->size() is the number of commands found and path[i] is the
length of the i-th command (copy length plus insert length).
Note that the sum of the lengths of all commands can be less than num_bytes.
On return, the nodes[0..num_bytes] array will have the following
"ZopfliNode array invariant":
For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then
(1) nodes[i].copy_length() >= 2
(2) nodes[i].command_length() <= i and
(3) nodes[i - nodes[i].command_length()].cost < kInfinity */
BROTLI_INTERNAL size_t BrotliZopfliComputeShortestPath(
MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params, const size_t max_backward_limit,
const int* dist_cache, HasherHandle hasher, ZopfliNode* nodes);
BROTLI_INTERNAL void BrotliZopfliCreateCommands(const size_t num_bytes,
const size_t block_start,
const size_t max_backward_limit,
const ZopfliNode* nodes,
int* dist_cache,
size_t* last_insert_len,
Command* commands,
size_t* num_literals);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_ */

143
modules/brotli/enc/backward_references_inc.h Normal file
View File

@@ -0,0 +1,143 @@
/* NOLINT(build/header_guard) */
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* template parameters: FN */
static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
size_t num_bytes, size_t position,
const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
size_t* last_insert_len, Command* commands, size_t* num_commands,
size_t* num_literals) {
/* Set maximum distance, see section 9.1. of the spec. */
const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
const Command* const orig_commands = commands;
size_t insert_length = *last_insert_len;
const size_t pos_end = position + num_bytes;
const size_t store_end = num_bytes >= FN(StoreLookahead)() ?
position + num_bytes - FN(StoreLookahead)() + 1 : position;
/* For speed up heuristics for random data. */
const size_t random_heuristics_window_size =
LiteralSpreeLengthForSparseSearch(params);
size_t apply_random_heuristics = position + random_heuristics_window_size;
/* Minimum score to accept a backward reference. */
const score_t kMinScore = BROTLI_SCORE_BASE + 100;
FN(PrepareDistanceCache)(hasher, dist_cache);
while (position + FN(HashTypeLength)() < pos_end) {
size_t max_length = pos_end - position;
size_t max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
HasherSearchResult sr;
sr.len = 0;
sr.len_x_code = 0;
sr.distance = 0;
sr.score = kMinScore;
if (FN(FindLongestMatch)(hasher, dictionary, dictionary_hash,
ringbuffer, ringbuffer_mask, dist_cache,
position, max_length, max_distance, &sr)) {
/* Found a match. Let's look for something even better ahead. */
int delayed_backward_references_in_row = 0;
--max_length;
for (;; --max_length) {
const score_t cost_diff_lazy = 175;
BROTLI_BOOL is_match_found;
HasherSearchResult sr2;
sr2.len = params->quality < MIN_QUALITY_FOR_EXTENSIVE_REFERENCE_SEARCH ?
BROTLI_MIN(size_t, sr.len - 1, max_length) : 0;
sr2.len_x_code = 0;
sr2.distance = 0;
sr2.score = kMinScore;
max_distance = BROTLI_MIN(size_t, position + 1, max_backward_limit);
is_match_found = FN(FindLongestMatch)(hasher, dictionary,
dictionary_hash, ringbuffer, ringbuffer_mask, dist_cache,
position + 1, max_length, max_distance, &sr2);
if (is_match_found && sr2.score >= sr.score + cost_diff_lazy) {
/* Ok, let's just write one byte for now and start a match from the
next byte. */
++position;
++insert_length;
sr = sr2;
if (++delayed_backward_references_in_row < 4 &&
position + FN(HashTypeLength)() < pos_end) {
continue;
}
}
break;
}
apply_random_heuristics =
position + 2 * sr.len + random_heuristics_window_size;
max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
{
/* The first 16 codes are special short-codes,
and the minimum offset is 1. */
size_t distance_code =
ComputeDistanceCode(sr.distance, max_distance, dist_cache);
if (sr.distance <= max_distance && distance_code > 0) {
dist_cache[3] = dist_cache[2];
dist_cache[2] = dist_cache[1];
dist_cache[1] = dist_cache[0];
dist_cache[0] = (int)sr.distance;
FN(PrepareDistanceCache)(hasher, dist_cache);
}
InitCommand(commands++, insert_length, sr.len, sr.len ^ sr.len_x_code,
distance_code);
}
*num_literals += insert_length;
insert_length = 0;
/* Put the hash keys into the table, if there are enough bytes left.
Depending on the hasher implementation, it can push all positions
in the given range or only a subset of them. */
FN(StoreRange)(hasher, ringbuffer, ringbuffer_mask, position + 2,
BROTLI_MIN(size_t, position + sr.len, store_end));
position += sr.len;
} else {
++insert_length;
++position;
/* If we have not seen matches for a long time, we can skip some
match lookups. Unsuccessful match lookups are very very expensive
and this kind of a heuristic speeds up compression quite
a lot. */
if (position > apply_random_heuristics) {
/* Going through uncompressible data, jump. */
if (position >
apply_random_heuristics + 4 * random_heuristics_window_size) {
/* It is quite a long time since we saw a copy, so we assume
that this data is not compressible, and store hashes less
often. Hashes of non compressible data are less likely to
turn out to be useful in the future, too, so we store less of
them to not to flood out the hash table of good compressible
data. */
const size_t kMargin =
BROTLI_MAX(size_t, FN(StoreLookahead)() - 1, 4);
size_t pos_jump =
BROTLI_MIN(size_t, position + 16, pos_end - kMargin);
for (; position < pos_jump; position += 4) {
FN(Store)(hasher, ringbuffer, ringbuffer_mask, position);
insert_length += 4;
}
} else {
const size_t kMargin =
BROTLI_MAX(size_t, FN(StoreLookahead)() - 1, 2);
size_t pos_jump =
BROTLI_MIN(size_t, position + 8, pos_end - kMargin);
for (; position < pos_jump; position += 2) {
FN(Store)(hasher, ringbuffer, ringbuffer_mask, position);
insert_length += 2;
}
}
}
}
}
insert_length += pos_end - position;
*last_insert_len = insert_length;
*num_commands += (size_t)(commands - orig_commands);
}

35
modules/brotli/enc/bit_cost.c Normal file
View File

@@ -0,0 +1,35 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Functions to estimate the bit cost of Huffman trees. */
#include "./bit_cost.h"
#include "../common/constants.h"
#include <brotli/types.h>
#include "./fast_log.h"
#include "./histogram.h"
#include "./port.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
#define FN(X) X ## Literal
#include "./bit_cost_inc.h" /* NOLINT(build/include) */
#undef FN
#define FN(X) X ## Command
#include "./bit_cost_inc.h" /* NOLINT(build/include) */
#undef FN
#define FN(X) X ## Distance
#include "./bit_cost_inc.h" /* NOLINT(build/include) */
#undef FN
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

144
modules/brotli/enc/bit_cost.h
View File

@@ -4,19 +4,22 @@
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
// Functions to estimate the bit cost of Huffman trees.
/* Functions to estimate the bit cost of Huffman trees. */
#ifndef BROTLI_ENC_BIT_COST_H_
#define BROTLI_ENC_BIT_COST_H_
#include "./entropy_encode.h"
#include <brotli/types.h>
#include "./fast_log.h"
#include "./types.h"
#include "./histogram.h"
#include "./port.h"
namespace brotli {
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
static inline double ShannonEntropy(const uint32_t *population, size_t size,
size_t *total) {
static BROTLI_INLINE double ShannonEntropy(const uint32_t *population,
size_t size, size_t *total) {
size_t sum = 0;
double retval = 0;
const uint32_t *population_end = population + size;
@@ -27,135 +30,34 @@ static inline double ShannonEntropy(const uint32_t *population, size_t size,
while (population < population_end) {
p = *population++;
sum += p;
retval -= static_cast<double>(p) * FastLog2(p);
retval -= (double)p * FastLog2(p);
odd_number_of_elements_left:
p = *population++;
sum += p;
retval -= static_cast<double>(p) * FastLog2(p);
retval -= (double)p * FastLog2(p);
}
if (sum) retval += static_cast<double>(sum) * FastLog2(sum);
if (sum) retval += (double)sum * FastLog2(sum);
*total = sum;
return retval;
}
static inline double BitsEntropy(const uint32_t *population, size_t size) {
static BROTLI_INLINE double BitsEntropy(
const uint32_t *population, size_t size) {
size_t sum;
double retval = ShannonEntropy(population, size, &sum);
if (retval < sum) {
// At least one bit per literal is needed.
retval = static_cast<double>(sum);
/* At least one bit per literal is needed. */
retval = (double)sum;
}
return retval;
}
template<int kSize>
double PopulationCost(const Histogram<kSize>& histogram) {
static const double kOneSymbolHistogramCost = 12;
static const double kTwoSymbolHistogramCost = 20;
static const double kThreeSymbolHistogramCost = 28;
static const double kFourSymbolHistogramCost = 37;
if (histogram.total_count_ == 0) {
return kOneSymbolHistogramCost;
}
int count = 0;
int s[5];
for (int i = 0; i < kSize; ++i) {
if (histogram.data_[i] > 0) {
s[count] = i;
++count;
if (count > 4) break;
}
}
if (count == 1) {
return kOneSymbolHistogramCost;
}
if (count == 2) {
return (kTwoSymbolHistogramCost +
static_cast<double>(histogram.total_count_));
}
if (count == 3) {
const uint32_t histo0 = histogram.data_[s[0]];
const uint32_t histo1 = histogram.data_[s[1]];
const uint32_t histo2 = histogram.data_[s[2]];
const uint32_t histomax = std::max(histo0, std::max(histo1, histo2));
return (kThreeSymbolHistogramCost +
2 * (histo0 + histo1 + histo2) - histomax);
}
if (count == 4) {
uint32_t histo[4];
for (int i = 0; i < 4; ++i) {
histo[i] = histogram.data_[s[i]];
}
// Sort
for (int i = 0; i < 4; ++i) {
for (int j = i + 1; j < 4; ++j) {
if (histo[j] > histo[i]) {
std::swap(histo[j], histo[i]);
}
}
}
const uint32_t h23 = histo[2] + histo[3];
const uint32_t histomax = std::max(h23, histo[0]);
return (kFourSymbolHistogramCost +
3 * h23 + 2 * (histo[0] + histo[1]) - histomax);
}
BROTLI_INTERNAL double BrotliPopulationCostLiteral(const HistogramLiteral*);
BROTLI_INTERNAL double BrotliPopulationCostCommand(const HistogramCommand*);
BROTLI_INTERNAL double BrotliPopulationCostDistance(const HistogramDistance*);
// In this loop we compute the entropy of the histogram and simultaneously
// build a simplified histogram of the code length codes where we use the
// zero repeat code 17, but we don't use the non-zero repeat code 16.
double bits = 0;
size_t max_depth = 1;
uint32_t depth_histo[kCodeLengthCodes] = { 0 };
const double log2total = FastLog2(histogram.total_count_);
for (size_t i = 0; i < kSize;) {
if (histogram.data_[i] > 0) {
// Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
// = log2(total_count) - log2(count(symbol))
double log2p = log2total - FastLog2(histogram.data_[i]);
// Approximate the bit depth by round(-log2(P(symbol)))
size_t depth = static_cast<size_t>(log2p + 0.5);
bits += histogram.data_[i] * log2p;
if (depth > 15) {
depth = 15;
}
if (depth > max_depth) {
max_depth = depth;
}
++depth_histo[depth];
++i;
} else {
// Compute the run length of zeros and add the appropriate number of 0 and
// 17 code length codes to the code length code histogram.
uint32_t reps = 1;
for (size_t k = i + 1; k < kSize && histogram.data_[k] == 0; ++k) {
++reps;
}
i += reps;
if (i == kSize) {
// Don't add any cost for the last zero run, since these are encoded
// only implicitly.
break;
}
if (reps < 3) {
depth_histo[0] += reps;
} else {
reps -= 2;
while (reps > 0) {
++depth_histo[17];
// Add the 3 extra bits for the 17 code length code.
bits += 3;
reps >>= 3;
}
}
}
}
// Add the estimated encoding cost of the code length code histogram.
bits += static_cast<double>(18 + 2 * max_depth);
// Add the entropy of the code length code histogram.
bits += BitsEntropy(depth_histo, kCodeLengthCodes);
return bits;
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
} // namespace brotli
#endif // BROTLI_ENC_BIT_COST_H_
#endif /* BROTLI_ENC_BIT_COST_H_ */

127
modules/brotli/enc/bit_cost_inc.h Normal file
View File

@@ -0,0 +1,127 @@
/* NOLINT(build/header_guard) */
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* template parameters: FN */
#define HistogramType FN(Histogram)
double FN(BrotliPopulationCost)(const HistogramType* histogram) {
static const double kOneSymbolHistogramCost = 12;
static const double kTwoSymbolHistogramCost = 20;
static const double kThreeSymbolHistogramCost = 28;
static const double kFourSymbolHistogramCost = 37;
const size_t data_size = FN(HistogramDataSize)();
int count = 0;
size_t s[5];
double bits = 0.0;
size_t i;
if (histogram->total_count_ == 0) {
return kOneSymbolHistogramCost;
}
for (i = 0; i < data_size; ++i) {
if (histogram->data_[i] > 0) {
s[count] = i;
++count;
if (count > 4) break;
}
}
if (count == 1) {
return kOneSymbolHistogramCost;
}
if (count == 2) {
return (kTwoSymbolHistogramCost + (double)histogram->total_count_);
}
if (count == 3) {
const uint32_t histo0 = histogram->data_[s[0]];
const uint32_t histo1 = histogram->data_[s[1]];
const uint32_t histo2 = histogram->data_[s[2]];
const uint32_t histomax =
BROTLI_MAX(uint32_t, histo0, BROTLI_MAX(uint32_t, histo1, histo2));
return (kThreeSymbolHistogramCost +
2 * (histo0 + histo1 + histo2) - histomax);
}
if (count == 4) {
uint32_t histo[4];
uint32_t h23;
uint32_t histomax;
for (i = 0; i < 4; ++i) {
histo[i] = histogram->data_[s[i]];
}
/* Sort */
for (i = 0; i < 4; ++i) {
size_t j;
for (j = i + 1; j < 4; ++j) {
if (histo[j] > histo[i]) {
BROTLI_SWAP(uint32_t, histo, j, i);
}
}
}
h23 = histo[2] + histo[3];
histomax = BROTLI_MAX(uint32_t, h23, histo[0]);
return (kFourSymbolHistogramCost +
3 * h23 + 2 * (histo[0] + histo[1]) - histomax);
}
{
/* In this loop we compute the entropy of the histogram and simultaneously
build a simplified histogram of the code length codes where we use the
zero repeat code 17, but we don't use the non-zero repeat code 16. */
size_t max_depth = 1;
uint32_t depth_histo[BROTLI_CODE_LENGTH_CODES] = { 0 };
const double log2total = FastLog2(histogram->total_count_);
for (i = 0; i < data_size;) {
if (histogram->data_[i] > 0) {
/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
= log2(total_count) - log2(count(symbol)) */
double log2p = log2total - FastLog2(histogram->data_[i]);
/* Approximate the bit depth by round(-log2(P(symbol))) */
size_t depth = (size_t)(log2p + 0.5);
bits += histogram->data_[i] * log2p;
if (depth > 15) {
depth = 15;
}
if (depth > max_depth) {
max_depth = depth;
}
++depth_histo[depth];
++i;
} else {
/* Compute the run length of zeros and add the appropriate number of 0
and 17 code length codes to the code length code histogram. */
uint32_t reps = 1;
size_t k;
for (k = i + 1; k < data_size && histogram->data_[k] == 0; ++k) {
++reps;
}
i += reps;
if (i == data_size) {
/* Don't add any cost for the last zero run, since these are encoded
only implicitly. */
break;
}
if (reps < 3) {
depth_histo[0] += reps;
} else {
reps -= 2;
while (reps > 0) {
++depth_histo[BROTLI_REPEAT_ZERO_CODE_LENGTH];
/* Add the 3 extra bits for the 17 code length code. */
bits += 3;
reps >>= 3;
}
}
}
}
/* Add the estimated encoding cost of the code length code histogram. */
bits += (double)(18 + 2 * max_depth);
/* Add the entropy of the code length code histogram. */
bits += BitsEntropy(depth_histo, BROTLI_CODE_LENGTH_CODES);
}
return bits;
}
#undef HistogramType

33
modules/brotli/enc/block_encoder_inc.h Normal file
View File

@@ -0,0 +1,33 @@
/* NOLINT(build/header_guard) */
/* Copyright 2014 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* template parameters: FN */
#define HistogramType FN(Histogram)
/* Creates entropy codes for all block types and stores them to the bit
stream. */
static void FN(BuildAndStoreEntropyCodes)(MemoryManager* m, BlockEncoder* self,
const HistogramType* histograms, const size_t histograms_size,
HuffmanTree* tree, size_t* storage_ix, uint8_t* storage) {
const size_t alphabet_size = self->alphabet_size_;
const size_t table_size = histograms_size * alphabet_size;
self->depths_ = BROTLI_ALLOC(m, uint8_t, table_size);
self->bits_ = BROTLI_ALLOC(m, uint16_t, table_size);
if (BROTLI_IS_OOM(m)) return;
{
size_t i;
for (i = 0; i < histograms_size; ++i) {
size_t ix = i * alphabet_size;
BuildAndStoreHuffmanTree(&histograms[i].data_[0], alphabet_size, tree,
&self->depths_[ix], &self->bits_[ix], storage_ix, storage);
}
}
}
#undef HistogramType

Some files were not shown because too many files have changed in this diff Show More