Add script to clone source tree

Author: Ahrotahn

.cirrus.yml

@@ -3,12 +3,9 @@ container:
 
 code_check_task:
     pip_cache:
-        folder: ~/.cache/pip
+        folder: /usr/local/lib/python3.6/site-packages
         fingerprint_script: cat .cirrus_requirements.txt
         populate_script: pip install -r .cirrus_requirements.txt
-    pip_install_script:
-        # Needed in order for yapf to be fully installed
-        - pip install -r .cirrus_requirements.txt
     utils_script:
         - python3 -m yapf --style '.style.yapf' -e '*/third_party/*' -rpd utils
         - ./devutils/run_utils_pylint.py --hide-fixme
@@ -22,17 +19,32 @@ validate_config_task:
     validate_config_script: ./devutils/validate_config.py
 
 validate_with_source_task:
+    pip_cache:
+        folder: /usr/local/lib/python3.6/site-packages
+        fingerprint_script: cat .cirrus_requirements.txt
+        populate_script: pip install -r .cirrus_requirements.txt
     chromium_download_cache:
         folder: chromium_download_cache
         fingerprint_script: cat chromium_version.txt
-        populate_script:
-            # This directory will not exist when this is called, unless cach retrieval
+        populate_script: |
+            # These directories will not exist when this is called, unless cache retrieval
             # fails and leaves partially-complete files around.
-            - rm -rf chromium_download_cache || true
-            - mkdir chromium_download_cache
-            - ./utils/downloads.py retrieve -i downloads.ini -c chromium_download_cache
-    unpack_source_script:
-        - ./utils/downloads.py unpack -i downloads.ini -c chromium_download_cache chromium_src
+            rm -rf chromium_src
+            rm -rf chromium_download_cache
+            mkdir chromium_download_cache
+            # Attempt to download tarball
+            if ! ./utils/downloads.py retrieve -i downloads.ini -c chromium_download_cache ; then
+              # If tarball is not available, attempt a clone
+              ./utils/clone.py -o chromium_src
+              rm -rf chromium_src/uc_staging
+              find chromium_src -type d -name '.git' -exec rm -rf "{}" \; -prune
+              tar cf chromium_download_cache/chromium-$(cat chromium_version.txt).tar.xz \
+                --transform s/chromium_src/chromium-$(cat chromium_version.txt)/ chromium_src
+            fi
+    unpack_source_script: |
+        if [ ! -d chromium_src ]; then
+          ./utils/downloads.py unpack -i downloads.ini -c chromium_download_cache chromium_src
+        fi
     validate_patches_script:
         - ./devutils/validate_patches.py -l chromium_src
     validate_lists_script:

.cirrus_Dockerfile

@@ -1,5 +1,5 @@
 # Dockerfile for Python 3 with xz-utils (for tar.xz unpacking)
 
-FROM python:3.6-slim
+FROM python:3.6-slim-buster
 
-RUN apt update && apt install -y xz-utils patch axel
+RUN apt update && apt install -y xz-utils patch axel curl git

.cirrus_requirements.txt

@@ -3,5 +3,6 @@ astroid==2.1.0 # via pylint
 pylint==2.2.2
 pytest-cov==2.6.0
 pytest==3.10.1
+httplib2==0.11.3
 requests==2.21.0
 yapf==0.25.0

devutils/update_lists.py

@@ -22,6 +22,7 @@
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent / 'utils'))
 from _common import get_logger
 from domain_substitution import DomainRegexList, TREE_ENCODINGS
+from prune_binaries import CONTINGENT_PATHS
 sys.path.pop(0)
 
 # Encoding for output files
@@ -74,6 +75,7 @@
     '*.woff',
     '*.woff2',
     '*makefile',
+    '*.profdata',
     '*.xcf',
     '*.cur',
     '*.pdf',
@@ -253,22 +255,23 @@ def compute_lists_proc(path, source_tree, search_regex):
     symlink_set = set()
     if path.is_file():
         relative_path = path.relative_to(source_tree)
-        if path.is_symlink():
-            try:
-                resolved_relative_posix = path.resolve().relative_to(source_tree).as_posix()
-                symlink_set.add((resolved_relative_posix, relative_path.as_posix()))
-            except ValueError:
-                # Symlink leads out of the source tree
-                pass
-        else:
-            try:
-                if should_prune(path, relative_path, used_pep_set, used_pip_set):
-                    pruning_set.add(relative_path.as_posix())
-                elif should_domain_substitute(path, relative_path, search_regex, used_dep_set,
-                                              used_dip_set):
-                    domain_substitution_set.add(relative_path.as_posix())
-            except: #pylint: disable=bare-except
-                get_logger().exception('Unhandled exception while processing %s', relative_path)
+        if not any(cpath in str(relative_path.as_posix()) for cpath in CONTINGENT_PATHS):
+            if path.is_symlink():
+                try:
+                    resolved_relative_posix = path.resolve().relative_to(source_tree).as_posix()
+                    symlink_set.add((resolved_relative_posix, relative_path.as_posix()))
+                except ValueError:
+                    # Symlink leads out of the source tree
+                    pass
+            elif not any(skip in ('.git', '__pycache__', 'uc_staging') for skip in path.parts):
+                try:
+                    if should_prune(path, relative_path, used_pep_set, used_pip_set):
+                        pruning_set.add(relative_path.as_posix())
+                    elif should_domain_substitute(path, relative_path, search_regex, used_dep_set,
+                                                  used_dip_set):
+                        domain_substitution_set.add(relative_path.as_posix())
+                except: #pylint: disable=bare-except
+                    get_logger().exception('Unhandled exception while processing %s', relative_path)
     return (used_pep_set, used_pip_set, used_dep_set, used_dip_set, pruning_set,
             domain_substitution_set, symlink_set)
 

docs/developing.md

@@ -34,15 +34,22 @@ To gain a deeper understanding of this updating process, have a read through [do
     * This is available in most (if not all) Linux distributions, and also Homebrew on macOS.
     * This utility facilitates most of the updating process, so it is important to learn how to use this. The manpage for quilt (as of early 2017) lacks an example of a workflow. There are multiple guides online, but [this guide from Debian](https://wiki.debian.org/UsingQuilt) and [the referenced guide on that page](https://raphaelhertzog.com/2012/08/08/how-to-use-quilt-to-manage-patches-in-debian-packages/) are the ones referenced in developing the current workflow.
 * Python 3.6 or newer
+    * `httplib2` and `six` are also required if you wish to utilize a source clone instead of the source tarball.
 
 ### Downloading the source code
 
+#### Source tarball download (recommended):
 ```sh
 mkdir -p build/download_cache
 ./utils/downloads.py retrieve -i downloads.ini -c build/download_cache
 ./utils/downloads.py unpack -i downloads.ini -c build/download_cache build/src
 ```
 
+#### Source clone:
+```sh
+./utils/clone.py -o build/src
+```
+
 ### Updating lists
 
 The utility `devutils/update_lists.py` automates this process. By default, it will update the files in the local repo. Pass in `-h` or `--help` for available options.