Skip to content

Commit 1df9972

Browse files
PouyaF1AeroStun
authored andcommitted
[CMake] Cache plugin downloads
- Separate DownloadCacher runtime module - Fixed bugs in preprocessing - Fixed bugs in manifests processing - Tested appropriately Signed-off-by: AeroStun <24841307+AeroStun@users.noreply.github.com>
1 parent d67dbfa commit 1df9972

File tree

5 files changed

+189
-30
lines changed

5 files changed

+189
-30
lines changed

CMake/Modules/Resources.cmake

+1
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ macro (setup_smce_resources)
109109
copy_at_build (FILES "${PROJECT_SOURCE_DIR}/CMake/Scripts/ConfigureSketch.cmake" DESTINATION "${SMCE_RTRES_DIR}/SMCE/share/CMake/Scripts/")
110110
copy_at_build (FILES "${PROJECT_SOURCE_DIR}/CMake/Modules/BindGen.cmake" DESTINATION "${SMCE_RTRES_DIR}/SMCE/share/CMake/Modules/")
111111
copy_runtime_module (ArduinoPreludeVersion)
112+
copy_runtime_module (DownloadCacher)
112113
copy_runtime_module (InstallArduinoPrelude)
113114
copy_runtime_module (JuniperTranspile)
114115
copy_runtime_module (LegacyPreprocessing)

CMake/Runtime/DownloadCacher.cmake

+118
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#
2+
# DownloadCacher.cmake
3+
# Copyright 2021-2022 ItJustWorksTM
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
# function for cleaning the cache
19+
function (clear_download_cache)
20+
# delete all contents in the cached folder
21+
file (REMOVE_RECURSE "${SMCE_DIR}/cached_downloads/")
22+
message (STATUS "Cache cleared")
23+
endfunction ()
24+
25+
#[================================================================================================================[.rst:
26+
cached_download
27+
--------------------
28+
29+
Function to allow for caching of various downloads.
30+
31+
Usage:
32+
.. code-block:: cmake
33+
34+
cached_download (URL <url> DEST <dest-var> [RESULT_VARIABLE <res-var>] [FORCE_UPDATE])
35+
36+
Where ``<url>`` is the URL to the file to be downloaded, ``<dest-var>`` is the name of the variable to store the absolute
37+
real path to the download location passed to the parent scope by the function, ``<res-var>``
38+
is the name of the variable to store the result of all processes passed to the parent scope by
39+
the function, ``[FORCE_UPDATE]`` will define whether an already cached download should be re-downloaded and cached.
40+
41+
42+
Note:
43+
No additional arguments except for the ones defined are allowed in the function call.
44+
Uses SHA256 to create a uniquely hashed download location for each download.
45+
Download file is locked until the file has been downloaded and cached, this is done in order to avoid
46+
possible race conditions.
47+
#]================================================================================================================]
48+
function (cached_download)
49+
# initialize the cache download directory
50+
file (MAKE_DIRECTORY "${SMCE_DIR}/cached_downloads")
51+
52+
# parse args
53+
set (options FORCE_UPDATE)
54+
set (oneValueArgs URL RESULT_VARIABLE DEST)
55+
set (multiValueArgs)
56+
cmake_parse_arguments ("ARG" "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
57+
58+
if (${ARG_UNPARSED_ARGUMENTS})
59+
if (DEFINED ARG_RESULT_VARIABLE)
60+
set ("Function was called with too many arguments" PARENT_SCOPE)
61+
else ()
62+
message (FATAL_ERROR "Function was called with too many arguments")
63+
endif ()
64+
endif ()
65+
66+
# use SHA256 hash to URL to create unique identifier, lock download location mutex-style
67+
string (SHA256 hashed_dest "${ARG_URL}")
68+
set (download_path "${SMCE_DIR}/cached_downloads/${hashed_dest}")
69+
file (TOUCH "${download_path}.lock")
70+
file (LOCK "${download_path}.lock")
71+
72+
# check if plugin has already been downloaded and cached before
73+
if (EXISTS "${download_path}")
74+
set (index 1)
75+
else ()
76+
set (index -1)
77+
endif ()
78+
79+
# if download has been cached previously and is requested a forced re-download, clean previous download and re-cache
80+
if (${index} GREATER -1 AND ${ARG_FORCE_UPDATE})
81+
file (REMOVE "${download_path}")
82+
set (${index} -1)
83+
endif ()
84+
85+
# if download has not been cached, download. Otherwise pass.
86+
if (${index} LESS 0)
87+
message (DEBUG "Downloading")
88+
89+
file (DOWNLOAD "${ARG_URL}" "${download_path}" STATUS dlstatus)
90+
list (GET dlstatus 0 dlstatus_code)
91+
if (dlstatus_code)
92+
list (GET dlstatus 1 dlstatus_message)
93+
file (REMOVE "${download_path}")
94+
file (REMOVE "${download_path}.lock")
95+
if (DEFINED ARG_RESULT_VARIABLE)
96+
set ("${ARG_RESULT_VARIABLE}" "${dlstatus}" PARENT_SCOPE)
97+
else ()
98+
message (FATAL_ERROR "Download failed: (${dlstatus_code}) ${dlstatus_message}")
99+
endif ()
100+
endif ()
101+
message (DEBUG "Download complete")
102+
message (DEBUG "Cached!")
103+
else ()
104+
message (DEBUG "Has already been cached!")
105+
endif ()
106+
107+
# Unlock file and output absolute real path to download location
108+
file (LOCK "${download_path}.lock" RELEASE)
109+
file (REMOVE "${download_path}.lock")
110+
111+
if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.19")
112+
file (REAL_PATH "${download_path}" download_path)
113+
else ()
114+
get_filename_component (download_path "${download_path}" REALPATH)
115+
endif ()
116+
117+
set ("${ARG_DEST}" "${download_path}" PARENT_SCOPE)
118+
endfunction ()

CMake/Runtime/Preprocessing.cmake

+13-8
Original file line numberDiff line numberDiff line change
@@ -52,14 +52,8 @@ foreach (LIB ${SKETCH_LINK_LIBS})
5252
if (NOT TARGET "${LIB}")
5353
continue ()
5454
endif ()
55-
get_target_property (LIB_INCLUDE_DIRS "${LIB}" INCLUDE_DIRECTORIES)
56-
if (LIB_INCLUDE_DIRS)
57-
list (APPEND INCLUDE_DIRS ${LIB_INCLUDE_DIRS})
58-
endif ()
59-
get_target_property (LIB_COMP_DEFS "${LIB}" COMPILE_DEFINITIONS)
60-
if (LIB_COMP_DEFS)
61-
list (APPEND COMP_DEFS ${LIB_COMP_DEFS})
62-
endif ()
55+
get_target_property (target_type "${LIB}" TYPE)
56+
6357
get_target_property (LIB_INCLUDE_DIRS "${LIB}" INTERFACE_INCLUDE_DIRECTORIES)
6458
if (LIB_INCLUDE_DIRS)
6559
list (APPEND INCLUDE_DIRS ${LIB_INCLUDE_DIRS})
@@ -68,6 +62,17 @@ foreach (LIB ${SKETCH_LINK_LIBS})
6862
if (LIB_COMP_DEFS)
6963
list (APPEND COMP_DEFS ${LIB_COMP_DEFS})
7064
endif ()
65+
66+
if (NOT target_type STREQUAL "INTERFACE_LIBRARY")
67+
get_target_property (LIB_INCLUDE_DIRS "${LIB}" INCLUDE_DIRECTORIES)
68+
if (LIB_INCLUDE_DIRS)
69+
list (APPEND INCLUDE_DIRS ${LIB_INCLUDE_DIRS})
70+
endif ()
71+
get_target_property (LIB_COMP_DEFS "${LIB}" COMPILE_DEFINITIONS)
72+
if (LIB_COMP_DEFS)
73+
list (APPEND COMP_DEFS ${LIB_COMP_DEFS})
74+
endif ()
75+
endif()
7176
endforeach ()
7277

7378
set (INCDIR_FLAGS "")

CMake/Runtime/ProcessManifests.cmake

+9-16
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
# limitations under the License.
1616
#
1717

18+
include (DownloadCacher)
19+
1820
function (process_manifests)
1921
file (GLOB manifests "${CMAKE_SOURCE_DIR}/manifests/*.cmake")
2022

@@ -133,22 +135,10 @@ function (process_manifests)
133135
set (${patchu}root "${CMAKE_MATCH_1}")
134136

135137
elseif (PLUGIN${uPATCH}_URI MATCHES "^https?://(.+)$")
136-
message (STATUS "[Plugin ${PLUGIN_NAME}] Downloading${spatch}...")
137-
file (MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/plugins${upatch}_dls")
138-
set (ark${upatch}_path "${CMAKE_BINARY_DIR}/plugins${upatch}_dls/${PLUGIN_NAME}.ark")
139-
if (EXISTS "${ark${upatch}_path}")
140-
message (STATUS "[Plugin ${PLUGIN_NAME}] Download was cached")
141-
else ()
142-
file (DOWNLOAD "${CMAKE_MATCH_1}" "${ark${upatch}_path}" STATUS ark${upatch}_dlstatus)
143-
list (GET ark${upatch}_dlstatus 0 ark${upatch}_dlstatus_code)
144-
if (ark${upatch}_dlstatus_code)
145-
list (GET ark${upatch}_dlstatus 1 ark${upatch}_dlstatus_message)
146-
file (REMOVE "${ark${upatch}_path}")
147-
message (FATAL_ERROR "[Plugin ${PLUGIN_NAME}] Download failed: (${ark${upatch}_dlstatus_code}) ${ark${upatch}_dlstatus_message}")
148-
endif ()
149-
message (STATUS "[Plugin ${PLUGIN_NAME}] Download complete")
138+
cached_download (URL "${CMAKE_MATCH_1}" DEST ark${upatch}_path RESULT_VARIABLE error_param)
139+
if (error_param)
140+
message (FATAL_ERROR "${error_param}")
150141
endif ()
151-
152142
message (STATUS "[Plugin ${PLUGIN_NAME}] Inflating${spatch}...")
153143
set (${patchu}root "${CMAKE_BINARY_DIR}/plugins${upatch}_roots/${PLUGIN_NAME}")
154144
file (MAKE_DIRECTORY "${${patchu}root}")
@@ -293,14 +283,17 @@ function (process_manifests)
293283
if (sources)
294284
message (DEBUG "[Plugin ${PLUGIN_NAME}] Declaring OBJECT target")
295285
add_library (smce_plugin_${PLUGIN_NAME} OBJECT ${sources})
286+
target_include_directories (smce_plugin_${PLUGIN_NAME} SYSTEM PRIVATE
287+
"${SMCE_DIR}/RtResources/Ardrivo/include"
288+
"${PROJECT_BINARY_DIR}/SMCE_Devices/include"
289+
)
296290
set (visibility PUBLIC)
297291
else ()
298292
message (DEBUG "[Plugin ${PLUGIN_NAME}] Declaring INTERFACE target")
299293
add_library (smce_plugin_${PLUGIN_NAME} INTERFACE)
300294
set (visibility INTERFACE)
301295
endif ()
302296

303-
target_include_directories (smce_plugin_${PLUGIN_NAME} SYSTEM PRIVATE "${SMCE_DIR}/RtResources/Ardrivo/include" "${PROJECT_BINARY_DIR}/SMCE_Devices/include")
304297
target_include_directories (smce_plugin_${PLUGIN_NAME} ${visibility} ${incdirs})
305298
target_link_directories (smce_plugin_${PLUGIN_NAME} ${visibility} ${linkdirs})
306299
target_link_libraries (smce_plugin_${PLUGIN_NAME} ${visibility} ${linklibs})

test/LibManagement.cpp

+48-6
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ TEST_CASE("Valid manifests processing", "[Plugin]") {
9999
generator_override ? generator_override : (!bp::search_path("ninja").empty() ? "Ninja" : "");
100100
#endif
101101

102-
constexpr auto module_path = SMCE_PATH "/RtResources/SMCE/share/CMake/Modules/ProcessManifests.cmake";
102+
constexpr auto module_path = SMCE_PATH "/RtResources/SMCE/share/CMake/Modules/";
103103
const std::filesystem::path tmproot = SMCE_PATH "/tmp";
104104

105105
const auto id = smce::Uuid::generate();
@@ -150,23 +150,65 @@ TEST_CASE("Valid manifests processing", "[Plugin]") {
150150
empty_source << "# empty\n";
151151
std::ofstream loader{base_dir / "CMakeLists.txt"};
152152
loader << "cmake_minimum_required (VERSION 3.12)\n";
153+
loader << "list (APPEND CMAKE_MODULE_PATH \"" << module_path << "\")\n";
153154
loader << "project (Test)\n";
154155
loader << "add_library (Ardrivo INTERFACE)\n";
155156
loader << "add_executable (Sketch empty.cxx)\n";
156-
loader << "include (\"" << module_path << "\")\n";
157+
loader << "include (ProcessManifests)\n";
157158
loader << cmake_require_target("smce_plugin_ESP32_AnalogWrite");
158159
}
159160

160-
const auto res =
161-
bp::system(bp::shell, bp::start_dir(base_dir.generic_string()),
161+
const auto res = bp::system(bp::shell, bp::start_dir(base_dir.generic_string()),
162162
#if !BOOST_OS_WINDOWS
163-
bp::env["CMAKE_GENERATOR"] = generator,
163+
bp::env["CMAKE_GENERATOR"] = generator,
164164
#endif
165-
"cmake", "--log-level=DEBUG", "-S", ".", "-B", "build", (bp::std_out & bp::std_err) > stderr);
165+
"cmake", "--log-level=DEBUG", "-DSMCE_DIR=" SMCE_PATH, "-S", ".", "-B", "build",
166+
(bp::std_out & bp::std_err) > stderr);
166167
REQUIRE(res == 0);
167168
}
168169
}
169170

171+
TEST_CASE("Valid download caching", "[Plugin]") {
172+
smce::Toolchain tc{SMCE_PATH};
173+
REQUIRE(!tc.check_suitable_environment());
174+
175+
const auto cached_downloads_path = SMCE_PATH "/cached_downloads/";
176+
177+
if (std::filesystem::exists(cached_downloads_path)) {
178+
[[maybe_unused]] std::error_code ec;
179+
std::filesystem::remove_all(cached_downloads_path, ec);
180+
}
181+
182+
constexpr std::array versions_and_uris{
183+
std::pair{"0.2"sv, "https://github.com/ERROPiX/ESP32_AnalogWrite/archive/refs/tags/0.2.zip"sv},
184+
std::pair{"0.1"sv, "https://github.com/ERROPiX/ESP32_AnalogWrite/archive/refs/tags/0.1.zip"sv},
185+
std::pair{"0.2"sv, "https://github.com/ERROPiX/ESP32_AnalogWrite/archive/refs/tags/0.2.zip"sv}};
186+
187+
for (const auto& [version, uri] : versions_and_uris) {
188+
std::vector<smce::PluginManifest> plugins{
189+
smce::PluginManifest{.name = "ESP32_AnalogWrite",
190+
.version = std::string{version},
191+
.uri = std::string{uri},
192+
.defaults = smce::PluginManifest::Defaults::none}};
193+
194+
smce::SketchConfig skc{.fqbn = "arduino:avr:nano", .plugins = std::move(plugins)};
195+
196+
smce::Sketch sk{SKETCHES_PATH "noop", std::move(skc)};
197+
198+
const auto ec = tc.compile(sk);
199+
if (ec)
200+
std::cerr << tc.build_log().second;
201+
202+
REQUIRE_FALSE(ec);
203+
}
204+
205+
[[maybe_unused]] std::error_code ec;
206+
std::size_t count = std::filesystem::remove_all(cached_downloads_path, ec) - 1;
207+
REQUIRE_FALSE(ec);
208+
209+
REQUIRE(count == 2);
210+
}
211+
170212
#if SMCE_ARDRIVO_MQTT
171213

172214
TEST_CASE("Board remote preproc lib", "[Board]") {

0 commit comments

Comments
 (0)