Merge branch develop into main [2023_09_07] (#175)

* Agent: Support for building an Ubuntu Core snap.
* Support Ubuntu 22.04 and Debian 11 builds.
* Agent: Replace boost::filesystem usage with std::filesystem.
* SDK (Windows): Preview additional methods for partial downloads, streaming downloads, and download properties.
* bootstrap script determines OS and version on its own.
* Fix issue of non-existent /include path during bootstrap and/or build.

Author: shishirb-MSFT

.gitignore

@@ -56,3 +56,7 @@ _deps
 *_CPack_Packages/
 *.deb
 out/
+
+# Ubuntu Snap files
+*.snap
+snap/

CMakeLists.txt

@@ -15,10 +15,7 @@ option (DO_INCLUDE_PLUGINS "Build subproject plugins" OFF)
 option (DO_INCLUDE_SDK "Build subproject sdk-cpp" OFF)
 
 option (DO_BUILD_TESTS "Set DO_BUILD_TESTS to OFF to skip building tests." ON)
-
-set(CMAKE_CXX_STANDARD 14)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
-set(CMAKE_CXX_EXTENSIONS OFF)
+option (DO_BUILD_FOR_SNAP "Enable DO Snap build option" OFF)
 
 # Get verbose output from cmake generation and build steps
 set(CMAKE_VERBOSE_MAKEFILE ON)
@@ -61,6 +58,16 @@ else()
     message(FATAL_ERROR "Unknown platform for client")
 endif()
 
+# C++17 is required to use std::filesystem but Edge builds on Windows is not ready to move up yet.
+# Luckily, we do not need std::filesystem on Windows.
+if (DO_INCLUDE_SDK AND DO_PLATFORM_WINDOWS)
+    set(CMAKE_CXX_STANDARD 14)
+else ()
+    set(CMAKE_CXX_STANDARD 17)
+endif ()
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
 # PIC (Position Independent Code) ensures .a files can be linked to executables that have PIE enabled
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
@@ -82,7 +89,14 @@ if (DO_PLATFORM_LINUX)
     # (Enable --trace-expand in cmake and look at lines from boost_headers-config.cmake and its callers.)
     # Workaround: force cmake to look in "/usr" since the /lib and /bin paths are either
     # not relevant for our usage or they are symlinks to /usr/lib and /usr/bin.
-    set(CMAKE_PREFIX_PATH "/usr")
+    if (IS_SYMLINK "/lib" AND NOT EXISTS "/include")
+        if (CMAKE_PREFIX_PATH)
+            message (WARNING "/lib is a symlink and /include does not exist. CMAKE_PREFIX_PATH is already set, not overriding it.")
+        else ()
+            message (WARNING "/lib is a symlink and /include does not exist. Updating CMAKE_PREFIX_PATH to /usr.")
+            set(CMAKE_PREFIX_PATH "/usr")
+        endif ()
+    endif ()
 endif (DO_PLATFORM_LINUX)
 
 if (DO_PLATFORM_WINDOWS AND DO_INCLUDE_SDK)

README.md

@@ -102,7 +102,7 @@ This will handle installing to the appropriate paths, and also the necessary set
 > cd /tmp/build-deliveryoptimization-agent/linux-debug/
 > sudo apt-get install ./deliveryoptimization-agent*.deb
 > cd /tmp/build-deliveryoptimization-plugin-apt/linux-debug/
-> sudo apt get install ./deliveryoptimization-plugin-apt*.deb
+> sudo apt-get install ./deliveryoptimization-plugin-apt*.deb
 ```
 
 2. If you build and install using cmake, or through some other custom means, be sure to setup the DO user/groups correctly in your installation.

azure-pipelines/build/linux/du/docker/doclient-lite-docker.yml

@@ -33,13 +33,31 @@ jobs:
 
 - template: ../templates/do-docker-jobs.yml
   parameters:
-    targetOsArch: 'debian10_arm64'
+    targetOsArch: 'debian10_arm64' # azure pipelines does not support '.' in display names
     imageVersion: ${{variables.containerImageVersion}}
     stepsTemplate: 'doclient-lite-docker-steps.yml'
 
 - template: ../templates/do-docker-jobs.yml
   parameters:
-    targetOsArch: 'debian10_x64'
+    targetOsArch: 'debian10_x64' # azure pipelines does not support '.' in display names
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'doclient-lite-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'debian11_arm32' # azure pipelines does not support '.' in display names
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'doclient-lite-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'debian11_arm64' # azure pipelines does not support '.' in display names
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'doclient-lite-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'debian11_x64' # azure pipelines does not support '.' in display names
     imageVersion: ${{variables.containerImageVersion}}
     stepsTemplate: 'doclient-lite-docker-steps.yml'
 
@@ -61,3 +79,14 @@ jobs:
     imageVersion: ${{variables.containerImageVersion}}
     stepsTemplate: 'doclient-lite-docker-steps.yml'
 
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'ubuntu2204_arm64' # azure pipelines does not support '.' in display names
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'doclient-lite-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'ubuntu2204_x64' # azure pipelines does not support '.' in display names
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'doclient-lite-docker-steps.yml'

azure-pipelines/build/linux/du/docker/dopapt-docker.yml

@@ -43,6 +43,24 @@ jobs:
     imageVersion: ${{variables.containerImageVersion}}
     stepsTemplate: 'dopapt-docker-steps.yml'
 
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'debian11_arm32'
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'dopapt-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'debian11_arm64'
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'dopapt-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'debian11_x64'
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'dopapt-docker-steps.yml'
+
 - template: ../templates/do-docker-jobs.yml
   parameters:
     targetOsArch: 'ubuntu1804_arm64'
@@ -61,3 +79,14 @@ jobs:
     imageVersion: ${{variables.containerImageVersion}}
     stepsTemplate: 'dopapt-docker-steps.yml'
 
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'ubuntu2204_arm64'
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'dopapt-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'ubuntu2204_x64'
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'dopapt-docker-steps.yml'

azure-pipelines/build/linux/du/docker/dosdkcpp-docker.yml

@@ -43,6 +43,24 @@ jobs:
     imageVersion: ${{variables.containerImageVersion}}
     stepsTemplate: 'dosdkcpp-docker-steps.yml'
 
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'debian11_arm32'
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'dosdkcpp-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'debian11_arm64'
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'dosdkcpp-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'debian11_x64'
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'dosdkcpp-docker-steps.yml'
+
 - template: ../templates/do-docker-jobs.yml
   parameters:
     targetOsArch: 'ubuntu1804_arm64'
@@ -60,3 +78,15 @@ jobs:
     targetOsArch: 'ubuntu2004_x64'
     imageVersion: ${{variables.containerImageVersion}}
     stepsTemplate: 'dosdkcpp-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'ubuntu2204_arm64'
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'dosdkcpp-docker-steps.yml'
+
+- template: ../templates/do-docker-jobs.yml
+  parameters:
+    targetOsArch: 'ubuntu2204_x64'
+    imageVersion: ${{variables.containerImageVersion}}
+    stepsTemplate: 'dosdkcpp-docker-steps.yml'

azure-pipelines/build/linux/du/templates/doclient-lite-docker-steps.yml

@@ -14,14 +14,14 @@ steps:
   displayName: Login to ACR
   inputs:
     command: login
-    containerRegistry: doclientcontainerregistry
+    containerRegistry: doclient-dockercontainerregistry-buildpipeline   # name of the service connection that connect the pipeline to the ACR   
     repository: $(parameters.targetOsArch)
 
 - task: Docker@2
   displayName: Pull latest build image
   inputs:
     command: pull
-    containerRegistry: doclientcontainerregistry
+    containerRegistry: doclient-dockercontainerregistry-buildpipeline   # name of the service connection that connect the pipeline to the ACR
     arguments: 'doclientcontainerregistry.azurecr.io/${{parameters.targetOsArch}}:${{parameters.imageVersion}}'
 
 - task: CmdLine@2

azure-pipelines/build/linux/du/templates/doclient-lite-native-steps.yml

@@ -24,7 +24,7 @@ steps:
   inputs:
     targetType: 'filePath'
     filePath: 'build/scripts/check_binary_size.sh'
-    arguments: '1251928 /tmp/build-deliveryoptimization-agent/linux-${{parameters.config}}/client-lite/deliveryoptimization-agent'
+    arguments: '523672 /tmp/build-deliveryoptimization-agent/linux-${{parameters.config}}/client-lite/deliveryoptimization-agent'
   displayName: 'Limit binary size increase'
 
 - task: CmdLine@2

azure-pipelines/build/linux/du/templates/dopapt-docker-steps.yml

@@ -14,14 +14,14 @@ steps:
   displayName: Login to ACR
   inputs:
     command: login
-    containerRegistry: doclientcontainerregistry
+    containerRegistry: doclient-dockercontainerregistry-buildpipeline   # name of the service connection that connect the pipeline to the ACR
     repository: $(parameters.targetOsArch)
 
 - task: Docker@2
   displayName: Pull latest build image
   inputs:
     command: pull
-    containerRegistry: doclientcontainerregistry
+    containerRegistry: doclient-dockercontainerregistry-buildpipeline   # name of the service connection that connect the pipeline to the ACR
     arguments: 'doclientcontainerregistry.azurecr.io/${{parameters.targetOsArch}}:${{parameters.imageVersion}}'
 
 - task: CmdLine@2

azure-pipelines/build/linux/du/templates/dosdkcpp-docker-steps.yml

@@ -14,14 +14,14 @@ steps:
   displayName: Login to ACR
   inputs:
     command: login
-    containerRegistry: doclientcontainerregistry
+    containerRegistry: doclient-dockercontainerregistry-buildpipeline   # name of the service connection that connect the pipeline to the ACR
     repository: $(parameters.targetOsArch)
 
 - task: Docker@2
   displayName: Pull latest build image
   inputs:
     command: pull
-    containerRegistry: doclientcontainerregistry
+    containerRegistry: doclient-dockercontainerregistry-buildpipeline   # name of the service connection that connect the pipeline to the ACR
     arguments: 'doclientcontainerregistry.azurecr.io/${{parameters.targetOsArch}}:${{parameters.imageVersion}}'
 
 - task: CmdLine@2