diff --git a/helpers/DATA/firefox/process-json-files.py b/helpers/DATA/firefox/process-json-files.py
new file mode 100644
index 0000000000000000000000000000000000000000..5eb07f6de4faf595576afdc48ba09c26d52a47d8
--- /dev/null
+++ b/helpers/DATA/firefox/process-json-files.py
@@ -0,0 +1,238 @@
+#! /usr/bin/python3
+
+# Copyright (C) 2020, 2021 grizzlyuser <grizzlyuser@protonmail.com>
+# Copyright (C) 2020, 2021 Ruben Rodriguez <ruben@trisquel.info>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+
+import json
+import sys
+import time
+import copy
+import argparse
+import pathlib
+from collections import namedtuple
+from jsonschema import validate
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+ 'MAIN_PATH',
+ type=pathlib.Path,
+ help='path to main application source code directory')
+parser.add_argument(
+ 'BRANDING_PATH',
+ type=pathlib.Path,
+ help='path to branding source code directory')
+parser.add_argument(
+ '-i',
+ '--indent',
+ type=int,
+ default=2,
+ help='indent for pretty printing of output files')
+arguments = parser.parse_args()
+
+File = namedtuple('File', ['path', 'content'])
+
+
+class RemoteSettings:
+ DUMPS_PATH_RELATIVE = 'services/settings/dumps'
+ DUMPS_PATH_ABSOLUTE = arguments.MAIN_PATH / DUMPS_PATH_RELATIVE
+
+ _WRAPPER_NAME = 'data'
+
+ @classmethod
+ def wrap(cls, processed):
+ return File(processed.path, {cls._WRAPPER_NAME: processed.content})
+
+ @classmethod
+ def unwrap(cls, parsed_jsons):
+ return [File(json.path, json.content[cls._WRAPPER_NAME])
+ for json in parsed_jsons]
+
+ @classmethod
+ def should_modify_collection(cls, collection):
+ return True
+
+ @classmethod
+ def now(cls):
+ return int(round(time.time() / 10 ** 6))
+
+ @classmethod
+ def process_raw(cls, unwrapped_jsons, parsed_schema):
+ timestamps, result = [], []
+ for collection in unwrapped_jsons:
+ should_modify_collection = cls.should_modify_collection(collection)
+ for record in collection.content:
+ if should_modify_collection:
+ if cls.should_drop_record(record):
+ continue
+
+ clone = copy.deepcopy(record)
+
+ record = cls.process_record(record)
+
+ if clone != record:
+ timestamp = cls.now()
+ while timestamp in timestamps:
+ timestamp += 1
+ timestamps.append(timestamp)
+ record['last_modified'] = timestamp
+
+ if parsed_schema is not None:
+ validate(record, schema=parsed_schema)
+
+ result.append(record)
+
+ cls.OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
+
+ return File(cls.OUTPUT_PATH, result)
+
+ @classmethod
+ def process(cls, parsed_jsons, parsed_schema):
+ return cls.wrap(
+ cls.process_raw(
+ cls.unwrap(parsed_jsons),
+ parsed_schema))
+
+
+class Changes(RemoteSettings):
+ JSON_PATHS = tuple(RemoteSettings.DUMPS_PATH_ABSOLUTE.glob('*/*.json'))
+ OUTPUT_PATH = RemoteSettings.DUMPS_PATH_ABSOLUTE / 'monitor/changes.json'
+
+ @classmethod
+ def wrap(cls, processed):
+ return File(
+ processed.path, {
+ 'changes': processed.content, 'timestamp': cls.now()})
+
+ @classmethod
+ def process_raw(cls, unwrapped_jsons, parsed_schema):
+ changes = []
+
+ for collection in unwrapped_jsons:
+ if collection.path != RemoteSettings.DUMPS_PATH_ABSOLUTE / 'main/example.json':
+ latest_change = {}
+ latest_change['last_modified'] = max(
+ (record['last_modified'] for record in collection.content), default=0)
+ latest_change['bucket'] = collection.path.parent.name
+ latest_change['collection'] = collection.path.stem
+ changes.append(latest_change)
+
+ cls.OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
+
+ return File(cls.OUTPUT_PATH, changes)
+
+
+class SearchConfig(RemoteSettings):
+ JSON_PATHS = (
+ RemoteSettings.DUMPS_PATH_ABSOLUTE /
+ 'main/search-config.json',
+ )
+ SCHEMA_PATH = arguments.MAIN_PATH / \
+ 'toolkit/components/search/schema/search-engine-config-schema.json'
+ OUTPUT_PATH = JSON_PATHS[0]
+
+ _DUCKDUCKGO_SEARCH_ENGINE_ID = 'ddg@search.mozilla.org'
+
+ @classmethod
+ def should_drop_record(cls, search_engine):
+ return search_engine['webExtension']['id'] not in (
+ cls._DUCKDUCKGO_SEARCH_ENGINE_ID, 'wikipedia@search.mozilla.org')
+
+ @classmethod
+ def process_record(cls, search_engine):
+ [search_engine.pop(key, None)
+ for key in ['extraParams', 'telemetryId']]
+
+ general_specifier = {}
+ for specifier in search_engine['appliesTo'].copy():
+ if 'application' in specifier:
+ if 'distributions' in specifier['application']:
+ search_engine['appliesTo'].remove(specifier)
+ continue
+ specifier['application'].pop('extraParams', None)
+
+ if 'included' in specifier and 'everywhere' in specifier[
+ 'included'] and specifier['included']['everywhere']:
+ general_specifier = specifier
+
+ if not general_specifier:
+ general_specifier = {'included': {'everywhere': True}}
+ search_engine['appliesTo'].insert(0, general_specifier)
+ if search_engine['webExtension']['id'] == cls._DUCKDUCKGO_SEARCH_ENGINE_ID:
+ general_specifier['default'] = 'yes'
+
+ return search_engine
+
+
+class TippyTopSites:
+ JSON_PATHS = (
+ arguments.MAIN_PATH /
+ 'browser/components/newtab/data/content/tippytop/top_sites.json',
+ arguments.BRANDING_PATH /
+ 'tippytop/top_sites.json')
+
+ @classmethod
+ def process(cls, parsed_jsons, parsed_schema):
+ tippy_top_sites_main = parsed_jsons[0]
+ tippy_top_sites_branding = parsed_jsons[1]
+ result = tippy_top_sites_branding.content + \
+ [site for site in tippy_top_sites_main.content if 'wikipedia.org' in site['domains']]
+ return File(tippy_top_sites_main.path, result)
+
+
+class TopSites(RemoteSettings):
+ _TOP_SITES_JSON_PATH = 'main/top-sites.json'
+ _TOP_SITES_PATH_MAIN = RemoteSettings.DUMPS_PATH_ABSOLUTE / _TOP_SITES_JSON_PATH
+
+ JSON_PATHS = (
+ arguments.BRANDING_PATH /
+ RemoteSettings.DUMPS_PATH_RELATIVE /
+ _TOP_SITES_JSON_PATH,
+ _TOP_SITES_PATH_MAIN)
+ OUTPUT_PATH = _TOP_SITES_PATH_MAIN
+
+ @classmethod
+ def should_modify_collection(cls, collection):
+ return cls._TOP_SITES_PATH_MAIN == collection.path
+
+ @classmethod
+ def should_drop_record(cls, site):
+ return site['url'] != 'https://www.wikipedia.org/'
+
+ @classmethod
+ def process_record(cls, site):
+ site.pop('exclude_regions', None)
+ return site
+
+
+# To reflect the latest timestamps, Changes class should always come after
+# all other RemoteSettings subclasses
+processors = (SearchConfig, Changes)
+
+for processor in processors:
+ parsed_jsons = []
+ for json_path in processor.JSON_PATHS:
+ with json_path.open(encoding='utf-8') as file:
+ parsed_jsons.append(File(json_path, json.load(file)))
+
+ parsed_schema = None
+ if hasattr(processor, "SCHEMA_PATH"):
+ with processor.SCHEMA_PATH.open() as file:
+ parsed_schema = json.load(file)
+
+ processed = processor.process(parsed_jsons, parsed_schema)
+ with processed.path.open('w') as file:
+ json.dump(processed.content, file, indent=arguments.indent)
diff --git a/helpers/DATA/firefox/reprocess-search-config.py b/helpers/DATA/firefox/reprocess-search-config.py
deleted file mode 100644
index 4478cbb0ed55c17f30f77db1b260228cace2eb27..0000000000000000000000000000000000000000
--- a/helpers/DATA/firefox/reprocess-search-config.py
+++ /dev/null
@@ -1,52 +0,0 @@
-#! /usr/bin/python3
-
-# Copyright (C) 2020 Ruben Rodriguez <ruben@trisquel.info>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
-
-import json
-import sys
-
-data={}
-
-with open(sys.argv[1]) as f:
- data=json.load(f)
-
- whitelist=['ddg@search.mozilla.org','google@search.mozilla.org','wikipedia@search.mozilla.org','bing@search.mozilla.org']
-
- i=0
- newdata={"data":[]}
- trisquel={}
-
- for item in data["data"]:
- if item["webExtension"]["id"] in whitelist:
- item["appliesTo"][0]["included"]["everywhere"]=True
- item["appliesTo"][0]["default"]="no"
- if item["webExtension"]["id"] == 'ddg@search.mozilla.org':
- item["appliesTo"][0]["default"]="yes"
- item["appliesTo"][0]["orderHint"]=5000
- item["appliesTo"][0]["override"]=True
-# del item["appliesTo"][1]["application"]["distributions"]
-# del item["appliesTo"][1]["extraParams"]
-# del item["extraParams"]
- newdata["data"].append(item)
- i+=1
- trisquel={u'webExtension': {u'id': u'trisquel@search.mozilla.org'}, u'appliesTo': [{u'included': {u'everywhere': True}, 'default': 'no'}, {u'override': True, u'application': {'override': True, 'orderHint': 4000}}], u'id': u'4341e834-7290-4d33-beb0-377c04a49566', u'last_modified': 1595254832054, u'telemetryId': u'trisquel', u'schema': 1594312388241}
- ddghtml={u'webExtension': {u'id': u'ddg-html@search.mozilla.org'}, u'appliesTo': [{u'included': {u'everywhere': True}, 'default': 'no'}, {u'override': True, u'application': {'override': True, 'orderHint': 3000}}], u'id': u'55bf6437-3b82-41a6-98be-09c3b53b5b4d', u'last_modified': 1595254832054, u'telemetryId': u'ddg-html', u'schema': 1594312388241}
- newdata["data"].append(trisquel)
- newdata["data"].append(ddghtml)
-
-with open(sys.argv[1], 'w') as outfile:
- json.dump(newdata, outfile, indent=4)
diff --git a/helpers/make-firefox b/helpers/make-firefox
index 5247e76ba0c2b45eb948ae3e209d90bc95f4948c..b84b1b5fe4299ecfe0f0cd2d863e1bda63436962 100644
--- a/helpers/make-firefox
+++ b/helpers/make-firefox
@@ -1,6 +1,6 @@
#!/bin/bash
#
-# Copyright (C) 2008-2020 Ruben Rodriguez <ruben@trisquel.info>
+# Copyright (C) 2008-2021 Ruben Rodriguez <ruben@trisquel.info>
# Copyright (C) 2015 Santiago Rodriguez <santi@trisquel.info>
#
# This program is free software; you can redistribute it and/or modify
@@ -18,7 +18,7 @@
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
#
-VERSION=80
+VERSION=81
. ./config
@@ -206,7 +206,7 @@ sed 's|ddg@|ddg-html@|' -i browser/components/search/extensions/ddg-html/manifes
#sed '/search/s|q=|k1=-1\&kd=-1\&ko=1\&q=|' -i browser/components/search/extensions/ddg/manifest.json
# Reprocess search preconfiguration dump
-python $DATA/reprocess-search-config.py ./services/settings/dumps/main/search-config.json
+python3 $DATA/process-json-files.py . browser/components/extensions/schemas/
cat << EOF > debian/distribution.ini
[Global]