Skip to content

Commit 3dd5e7a

Browse files
gabrie30gabrie30
committed
Add preserve scm hostname flag
1 parent ca104dd commit 3dd5e7a

7 files changed

+230
-1
lines changed

‎cmd/clone.go

+16-1
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,10 @@ func cloneFunc(cmd *cobra.Command, argz []string) {
163163
os.Setenv("GHORG_GIT_FILTER", filter)
164164
}
165165

166+
if cmd.Flags().Changed("preserve-scm-hostname") {
167+
os.Setenv("GHORG_PRESERVE_SCM_HOSTNAME", "true")
168+
}
169+
166170
if cmd.Flags().Changed("skip-archived") {
167171
os.Setenv("GHORG_SKIP_ARCHIVED", "true")
168172
}
@@ -282,6 +286,10 @@ func cloneFunc(cmd *cobra.Command, argz []string) {
282286
os.Exit(1)
283287
}
284288

289+
if os.Getenv("GHORG_PRESERVE_SCM_HOSTNAME") == "true" {
290+
updateAbsolutePathToCloneToWithHostname()
291+
}
292+
285293
setOutputDirName(argz)
286294
setOuputDirAbsolutePath()
287295
args = argz
@@ -974,7 +982,14 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
974982
}
975983

976984
func writeGhorgStats(date string, allReposToCloneCount, cloneCount, pulledCount, cloneInfosCount, cloneErrorsCount, updateRemoteCount, newCommits, pruneCount int, hasCollisions bool) error {
977-
statsFilePath := filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), "_ghorg_stats.csv")
985+
var statsFilePath string
986+
absolutePath := os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO")
987+
if os.Getenv("GHORG_PRESERVE_SCM_HOSTNAME") == "true" {
988+
originalAbsolutePath := os.Getenv("GHORG_ORIGINAL_ABSOLUTE_PATH_TO_CLONE_TO")
989+
statsFilePath = filepath.Join(originalAbsolutePath, "_ghorg_stats.csv")
990+
} else {
991+
statsFilePath = filepath.Join(absolutePath, "_ghorg_stats.csv")
992+
}
978993

979994
fileExists := true
980995

‎cmd/root.go

+40
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package cmd
33
import (
44
"errors"
55
"fmt"
6+
"net/url"
67
"os"
78
"path/filepath"
89

@@ -68,6 +69,7 @@ var (
6869
quietMode bool
6970
noDirSize bool
7071
ghorgStatsEnabled bool
72+
ghorgPreserveScmHostname bool
7173
args []string
7274
cloneErrors []string
7375
cloneInfos []string
@@ -83,6 +85,40 @@ var rootCmd = &cobra.Command{
8385
},
8486
}
8587

88+
func getHostname() string {
89+
var hostname string
90+
baseURL := os.Getenv("GHORG_SCM_BASE_URL")
91+
if baseURL != "" {
92+
// Parse the URL to extract the hostname
93+
parsedURL, err := url.Parse(baseURL)
94+
if err != nil {
95+
colorlog.PrintError(fmt.Sprintf("Error parsing GHORG_SCM_BASE_URL clone may be affected, error: %v", err))
96+
}
97+
// Append the hostname to the absolute path
98+
hostname = parsedURL.Hostname()
99+
} else {
100+
// Use the predefined hostname based on the SCM type
101+
hostname = configs.GetScmTypeHostnames()
102+
}
103+
104+
return hostname
105+
}
106+
107+
// updateAbsolutePathToCloneToWithHostname modifies the absolute path by appending the hostname if the user has enabled it,
108+
// supporting the GHORG_PRESERVE_SCM_HOSTNAME feature. It checks the GHORG_PRESERVE_SCM_HOSTNAME environment variable, and if set to "true",
109+
// it uses the hostname from GHORG_SCM_BASE_URL if available, otherwise, it defaults to a predefined hostname based on the SCM type.
110+
func updateAbsolutePathToCloneToWithHostname() {
111+
// Verify if GHORG_PRESERVE_SCM_HOSTNAME is set to "true"
112+
if os.Getenv("GHORG_PRESERVE_SCM_HOSTNAME") == "true" {
113+
// Retrieve the hostname from the environment variable
114+
hostname := getHostname()
115+
absolutePath := os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO")
116+
os.Setenv("GHORG_ORIGINAL_ABSOLUTE_PATH_TO_CLONE_TO", absolutePath)
117+
absolutePath = filepath.Join(absolutePath, hostname)
118+
os.Setenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO", configs.EnsureTrailingSlashOnFilePath(absolutePath))
119+
}
120+
}
121+
86122
// reads in configuration file and updates anything not set to default
87123
func getOrSetDefaults(envVar string) {
88124
if envVar == "GHORG_COLOR" {
@@ -145,6 +181,8 @@ func getOrSetDefaults(envVar string) {
145181
os.Setenv(envVar, "owner")
146182
case "GHORG_BACKUP":
147183
os.Setenv(envVar, "false")
184+
case "GHORG_PRESERVE_SCM_HOSTNAME":
185+
os.Setenv(envVar, "false")
148186
case "GHORG_NO_TOKEN":
149187
os.Setenv(envVar, "false")
150188
case "GHORG_NO_DIR_SIZE":
@@ -228,6 +266,7 @@ func InitConfig() {
228266
getOrSetDefaults("GHORG_CLONE_PROTOCOL")
229267
getOrSetDefaults("GHORG_CLONE_TYPE")
230268
getOrSetDefaults("GHORG_SCM_TYPE")
269+
getOrSetDefaults("GHORG_PRESERVE_SCM_HOSTNAME")
231270
getOrSetDefaults("GHORG_SKIP_ARCHIVED")
232271
getOrSetDefaults("GHORG_SKIP_FORKS")
233272
getOrSetDefaults("GHORG_NO_CLEAN")
@@ -324,6 +363,7 @@ func init() {
324363
cloneCmd.Flags().BoolVar(&quietMode, "quiet", false, "GHORG_QUIET - Emit critical output only")
325364
cloneCmd.Flags().BoolVar(&includeSubmodules, "include-submodules", false, "GHORG_INCLUDE_SUBMODULES - Include submodules in all clone and pull operations.")
326365
cloneCmd.Flags().BoolVar(&ghorgStatsEnabled, "stats-enabled", false, "GHORG_STATS_ENABLED - Creates a CSV in the GHORG_ABSOLUTE_PATH_TO_CLONE_TO called _ghorg_stats.csv with info about each clone. This allows you to track clone data over time such as number of commits and size in megabytes of the clone directory.")
366+
cloneCmd.Flags().BoolVar(&ghorgPreserveScmHostname, "preserve-scm-hostname", false, "GHORG_PRESERVE_SCM_HOSTNAME - Appends the scm hostname to the GHORG_ABSOLUTE_PATH_TO_CLONE_TO which will organize your clones into specific folders by the scm provider. e.g. /github.com/kuberentes")
327367
cloneCmd.Flags().StringVarP(&baseURL, "base-url", "", "", "GHORG_SCM_BASE_URL - Change SCM base url, for on self hosted instances (currently gitlab, gitea and github (use format of https://git.mydomain.com/api/v3))")
328368
cloneCmd.Flags().StringVarP(&concurrency, "concurrency", "", "", "GHORG_CONCURRENCY - Max goroutines to spin up while cloning (default 25)")
329369
cloneCmd.Flags().StringVarP(&cloneDepth, "clone-depth", "", "", "GHORG_CLONE_DEPTH - Create a shallow clone with a history truncated to the specified number of commits")

‎configs/configs.go

+17
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"runtime"
1212
"strings"
1313

14+
"github.com/gabrie30/ghorg/colorlog"
1415
"github.com/gabrie30/ghorg/scm"
1516
"github.com/gabrie30/ghorg/utils"
1617

@@ -311,6 +312,22 @@ func VerifyTokenSet() error {
311312
return nil
312313
}
313314

315+
func GetScmTypeHostnames() string {
316+
switch os.Getenv("GHORG_SCM_TYPE") {
317+
case "github":
318+
return "github.com"
319+
case "gitlab":
320+
return "gitlab.com"
321+
case "gitea":
322+
return "gitea.com"
323+
case "bitbucket":
324+
return "bitbucket.com"
325+
default:
326+
colorlog.PrintErrorAndExit("Unsupported GHORG_SCM_TYPE")
327+
return ""
328+
}
329+
}
330+
314331
// VerifyConfigsSetCorrectly makes sure flags are set to appropriate values
315332
func VerifyConfigsSetCorrectly() error {
316333
scmType := os.Getenv("GHORG_SCM_TYPE")

‎scripts/bitbucket_cloud_integration_tests.sh

+11
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,14 @@ else
2929
echo "Fail: bitbucket org clone, commandline flags take overwrite conf.yaml"
3030
exit 1
3131
fi
32+
33+
# preserve scm hostname
34+
ghorg clone $BITBUCKET_WORKSPACE --token="${BITBUCKET_TOKEN}" --bitbucket-username="${BITBUCKET_USERNAME}" --path=/tmp --output-dir=testing_output_dir --scm=bitbucket --base-url="https://api.bitbucket.org/2.0" --preserve-scm-hostname
35+
36+
if [ -e /tmp/api.bitbucket.org/testing_output_dir ]
37+
then
38+
echo "Pass: bitbucket org clone, preserve scm hostname"
39+
else
40+
echo "Fail: bitbucket org clone, preserve scm hostname"
41+
exit 1
42+
fi

‎scripts/github_cloud_integration_tests.sh

+11
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,17 @@ else
2525
exit 1
2626
fi
2727

28+
# clone an org preserving scm hostname
29+
ghorg clone $GITHUB_ORG --token=$GITHUB_TOKEN --preserve-scm-hostname
30+
31+
if [ -e $HOME/ghorg/github.com/$GITHUB_ORG/$GHORG_TEST_REPO ]
32+
then
33+
echo "Pass: github org clone preserving scm hostname"
34+
else
35+
echo "Fail: github org clone preserving scm hostname"
36+
exit 1
37+
fi
38+
2839
# clone an org with no config file to a specific path
2940
ghorg clone $GITHUB_ORG --token=$GITHUB_TOKEN --path=/tmp --output-dir=testing_output_dir
3041

‎scripts/gitlab_cloud_integration_tests.sh

+41
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,17 @@ else
2222
exit 1
2323
fi
2424

25+
# preserve scm hostname
26+
ghorg clone $GITLAB_GROUP --token="${GITLAB_TOKEN}" --scm=gitlab --output-dir=examples-flat --preserve-scm-hostname
27+
28+
if [ -e "${HOME}"/ghorg/gitlab.com/examples-flat/microservice ]
29+
then
30+
echo "Pass: gitlab org clone flat file, preserve scm hostname"
31+
else
32+
echo "Fail: gitlab org clone flat file, preserve scm hostname"
33+
exit 1
34+
fi
35+
2536
#
2637
# TOP LEVEL GROUP TESTS
2738
#
@@ -114,6 +125,24 @@ else
114125
exit 1
115126
fi
116127

128+
ghorg clone $GITLAB_GROUP_2 --token="${GITLAB_TOKEN}" --scm=gitlab --clone-snippets --preserve-dir --preserve-scm-hostname
129+
130+
if [ -e "${HOME}"/ghorg/gitlab.com/"${GITLAB_GROUP_2}"/subgroup-2/foobar.snippets/test-snippet-2-3711655 ]
131+
then
132+
echo "Pass: gitlab group clone snippet 2 with preserve dir, preserve scm hostname"
133+
else
134+
echo "Fail: gitlab group clone snippet 2 with preserve dir, preserve scm hostname"
135+
exit 1
136+
fi
137+
138+
if [ -e "${HOME}"/ghorg/gitlab.com/"${GITLAB_GROUP_2}"/subgroup-2/foobar.snippets/test-snippet-1-3711654 ]
139+
then
140+
echo "Pass: gitlab group clone snippet 1 with preserve dir, preserve scm hostname"
141+
else
142+
echo "Fail: gitlab group clone snippet 1 with preserve dir, preserve scm hostname"
143+
exit 1
144+
fi
145+
117146
#
118147
# SUBGROUP TESTS
119148
#
@@ -153,6 +182,18 @@ else
153182
exit 1
154183
fi
155184

185+
# PRESERVE DIR, PRESERVE SCM HOSTNAME
186+
ghorg clone $GITLAB_GROUP/$GITLAB_SUB_GROUP --token="${GITLAB_TOKEN}" --scm=gitlab --preserve-dir
187+
188+
if [ -e "${HOME}"/ghorg/gitlab.com/"${GITLAB_GROUP}"/"${GITLAB_SUB_GROUP}"/wayne-industries/microservice ]
189+
then
190+
echo "Pass: gitlab subgroup clone preserve directories, preserve scm hostname"
191+
rm -rf "${HOME}/ghorg/gitlab.com/${GITLAB_GROUP}"
192+
else
193+
echo "Fail: gitlab subgroup clone preserve directories, preserve scm hostname"
194+
exit 1
195+
fi
196+
156197
# OUTPUT DIR AND PRESERVE DIR
157198
ghorg clone $GITLAB_GROUP/$GITLAB_SUB_GROUP --token="${GITLAB_TOKEN}" --scm=gitlab --preserve-dir --output-dir=examples-subgroup-preserve-output
158199

‎scripts/local-gitlab/integration-tests.sh

+94
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,79 @@ echo "CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR, SNIPPETS TEST FAILED
154154
exit 1
155155
fi
156156

157+
############ CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR, SNIPPETS, PERSERVE SCM HOSTNAME ############
158+
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="$TOKEN" --preserve-dir --output-dir=local-gitlab-v15-repos-snippets --clone-snippets --preserve-scm-hostname
159+
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="$TOKEN" --preserve-dir --output-dir=local-gitlab-v15-repos-snippets --clone-snippets --preserve-scm-hostname
160+
161+
GOT=$( ghorg ls gitlab.example.com/local-gitlab-v15-repos-snippets | grep -o 'gitlab.example.com/local-gitlab-v15-repos-snippets.*')
162+
WANT=$(cat <<EOF
163+
gitlab.example.com/local-gitlab-v15-repos-snippets/_ghorg_root_level_snippets
164+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group1
165+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group2
166+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group3
167+
EOF
168+
)
169+
170+
if [ "${WANT}" != "${GOT}" ]
171+
then
172+
echo "CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR, SNIPPETS, PRESERVE SCM HOSTNAME TEST FAILED local-gitlab-group1"
173+
exit 1
174+
fi
175+
176+
GOT=$( ghorg ls gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group1 | grep -o 'gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group1.*')
177+
WANT=$(cat <<EOF
178+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group1/baz0
179+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group1/baz1
180+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group1/baz2
181+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group1/baz3
182+
EOF
183+
)
184+
185+
if [ "${WANT}" != "${GOT}" ]
186+
then
187+
echo "CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR, SNIPPETS, PRESERVE SCM HOSTNAME TEST FAILED local-gitlab-group1"
188+
exit 1
189+
fi
190+
191+
GOT=$( ghorg ls gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group2 | grep -o 'gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group2.*')
192+
WANT=$(cat <<EOF
193+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group2/baz0
194+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group2/baz0.snippets
195+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group2/baz1
196+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group2/baz1.snippets
197+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group2/baz2
198+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group2/baz2.snippets
199+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group2/baz3
200+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group2/baz3.snippets
201+
EOF
202+
)
203+
204+
if [ "${WANT}" != "${GOT}" ]
205+
then
206+
echo "CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR, SNIPPETS, PRESERVE SCM HOSTNAME TEST FAILED local-gitlab-group2"
207+
exit 1
208+
fi
209+
210+
GOT=$( ghorg ls gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a | grep -o 'gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a.*')
211+
WANT=$(cat <<EOF
212+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a/subgroup-b
213+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a/subgroup_a_repo_0
214+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a/subgroup_a_repo_0.snippets
215+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a/subgroup_a_repo_1
216+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a/subgroup_a_repo_1.snippets
217+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a/subgroup_a_repo_2
218+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a/subgroup_a_repo_2.snippets
219+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a/subgroup_a_repo_3
220+
gitlab.example.com/local-gitlab-v15-repos-snippets/local-gitlab-group3/subgroup-a/subgroup_a_repo_3.snippets
221+
EOF
222+
)
223+
224+
if [ "${WANT}" != "${GOT}" ]
225+
then
226+
echo "CLONE AND TEST ALL-GROUPS, PRESERVE DIR, OUTPUT DIR, SNIPPETS, PRESERVE SCM HOSTNAME, TEST FAILED local-gitlab-group3/subgroup-a"
227+
exit 1
228+
fi
229+
157230
############ CLONE AND TEST ALL-GROUPS, OUTPUT DIR ############
158231
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-repos-flat
159232
ghorg clone all-groups --scm=gitlab --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-repos-flat
@@ -684,4 +757,25 @@ echo "CLONE AND TEST ALL-USERS, OUTPUT DIR SNIPPETS FAILED"
684757
exit 1
685758
fi
686759

760+
############ CLONE AND TEST ALL-USERS, OUTPUT DIR, SNIPPETS, PRESERVE SCM HOSTNAME ############
761+
ghorg clone all-users --scm=gitlab --clone-type=user --base-url="${GITLAB_URL}" --token="${TOKEN}" --output-dir=local-gitlab-v15-all-users-snippets --clone-snippets --preserve-scm-hostname
762+
763+
TEST_ALL_USERS_SNIPPETS_GOT=$(ghorg ls gitlab.example.com/local-gitlab-v15-all-users-snippets | grep -o 'gitlab.example.com/local-gitlab-v15-all-users-snippets.*')
764+
TEST_ALL_USERS_SNIPPETS_WANT=$(cat <<EOF
765+
gitlab.example.com/local-gitlab-v15-all-users-snippets/_ghorg_root_level_snippets
766+
gitlab.example.com/local-gitlab-v15-all-users-snippets/rootrepos0
767+
gitlab.example.com/local-gitlab-v15-all-users-snippets/rootrepos1
768+
gitlab.example.com/local-gitlab-v15-all-users-snippets/rootrepos2
769+
gitlab.example.com/local-gitlab-v15-all-users-snippets/rootrepos3
770+
gitlab.example.com/local-gitlab-v15-all-users-snippets/testuser1-repo
771+
gitlab.example.com/local-gitlab-v15-all-users-snippets/testuser1-repo.snippets
772+
EOF
773+
)
774+
775+
if [ "${TEST_ALL_USERS_SNIPPETS_WANT}" != "${TEST_ALL_USERS_SNIPPETS_GOT}" ]
776+
then
777+
echo "CLONE AND TEST ALL-USERS, OUTPUT DIR SNIPPETS, PRESERVE SCM HOSTNAME FAILED"
778+
exit 1
779+
fi
780+
687781
echo "INTEGRATOIN TESTS FINISHED..."

0 commit comments

Comments
 (0)