From cc7ea28ba2fb9aef21a5c7fdd0c62f03dac94f25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Radu=20Lucu=C8=9B?= Date: Wed, 9 Oct 2024 17:37:01 +0300 Subject: [PATCH] build(packagecloud): upload package to all distros and destroy older versions (#138) --- .github/workflows/release.yml | 7 +- packagecloud/main.go | 259 ++++++++++++++++++++++++++++++---- 2 files changed, 234 insertions(+), 32 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a873043..de65625 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -59,6 +59,8 @@ jobs: PACKAGECLOUD_USER: jsdelivr PACKAGECLOUD_REPO: globalping PACKAGECLOUD_APIKEY: ${{ secrets.PACKAGECLOUD_APIKEY }} + PACKAGECLOUD_MAX_DISTRO_VERSIONS_TO_SUPPORT: ${{ vars.PACKAGECLOUD_MAX_DISTRO_VERSIONS_TO_SUPPORT }} + PACKAGECLOUD_MAX_PACKAGE_VERSIONS_TO_KEEP: ${{ vars.PACKAGECLOUD_MAX_PACKAGE_VERSIONS_TO_KEEP }} steps: - uses: actions/checkout@v4 with: @@ -79,8 +81,9 @@ jobs: - run: echo "VERSION_NAME=${GITHUB_REF_NAME:1}" >> $GITHUB_ENV - run: ls -la - - run: go run packagecloud/main.go "globalping_${{ env.VERSION_NAME }}_linux_amd64.deb" "deb" - - run: go run packagecloud/main.go "globalping_${{ env.VERSION_NAME }}_linux_amd64.rpm" "rpm" + - run: go run packagecloud/main.go upload "globalping_${{ env.VERSION_NAME }}_linux_amd64.deb" "deb" + - run: go run packagecloud/main.go upload "globalping_${{ env.VERSION_NAME }}_linux_amd64.rpm" "rpm" + - run: go run packagecloud/main.go cleanup release_windows: needs: goreleaser diff --git a/packagecloud/main.go b/packagecloud/main.go index ad774ef..c93e670 100644 --- a/packagecloud/main.go +++ b/packagecloud/main.go @@ -2,67 +2,163 @@ package main import ( "bytes" + "encoding/json" "fmt" "io" "log" "mime/multipart" "net/http" "os" + "strconv" + "time" ) const ( - PackagecloudAPIURL = "https://packagecloud.io/api/v1/repos/" - PackagecloudDEBAny = "35" - PackagecloudRPMAny = "227" + PackagecloudAPIURL = "https://packagecloud.io" ) type Config struct { PackagecloudUser string PackagecloudRepo string PackagecloudAPIKey string + + MaxDistroVersionsToSupport int + MaxPackageVersionsToKeep int } func main() { - file := os.Args[1] - dist := os.Args[2] - if file == "" || dist == "" { - fmt.Println("Usage: upload ") - os.Exit(1) - } + cmd := os.Args[1] config := &Config{ PackagecloudUser: os.Getenv("PACKAGECLOUD_USER"), PackagecloudRepo: os.Getenv("PACKAGECLOUD_REPO"), PackagecloudAPIKey: os.Getenv("PACKAGECLOUD_APIKEY"), + + MaxDistroVersionsToSupport: 5, + MaxPackageVersionsToKeep: 3, + } + maxDistrosVersionsToSupport, _ := strconv.Atoi(os.Getenv("PACKAGECLOUD_MAX_DISTRO_VERSIONS_TO_SUPPORT")) + if maxDistrosVersionsToSupport != 0 { + config.MaxDistroVersionsToSupport = maxDistrosVersionsToSupport + } + maxPackageVersionsToKeep, _ := strconv.Atoi(os.Getenv("PACKAGECLOUD_MAX_PACKAGE_VERSIONS_TO_KEEP")) + if maxPackageVersionsToKeep != 0 { + config.MaxPackageVersionsToKeep = maxPackageVersionsToKeep + } + switch cmd { + case "upload": + upload(os.Args[2], os.Args[3], config) + case "cleanup": + cleanup(config) + default: + fmt.Println("Unknown command") + os.Exit(1) + } +} + +func upload(file string, dist string, config *Config) { + if file == "" || dist == "" { + fmt.Println("Usage: upload ") + os.Exit(1) + } + log.Println("Fetching distros") + distros, err := fetchDistros(config) + if err != nil { + log.Println("Failed to fetch distros: ", err) + os.Exit(1) + } + f, err := os.Open(file) + if err != nil { + log.Println("Failed to open file: ", err) + os.Exit(1) } + defer f.Close() switch dist { case "deb": - log.Printf("Uploading DEB package: %s\n", file) - err := uploadPackage(config, PackagecloudDEBAny, file) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - log.Println("DEB package uploaded") + uploadToAllDistros(config, f, distros.Deb) case "rpm": - log.Printf("Uploading RPM package: %s\n", file) - err := uploadPackage(config, PackagecloudRPMAny, file) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - log.Println("RPM package uploaded") + uploadToAllDistros(config, f, distros.Rpm) + default: fmt.Println("Unknown distro") os.Exit(1) } } -func uploadPackage(config *Config, distroVersionId string, path string) error { - var body bytes.Buffer - f, err := os.Open(path) +func uploadToAllDistros(config *Config, f *os.File, distros []*Distro) { + totalDistros := 0 + for _, distro := range distros { + l := len(distro.Versions) + for j := l - 1; (j >= 0) && (l-j <= config.MaxDistroVersionsToSupport); j-- { + totalDistros++ + log.Printf("Uploading to %s/%s (%d)\n", distro.IndexName, distro.Versions[j].IndexName, distro.Versions[j].ID) + err := uploadPackage(config, strconv.Itoa(distro.Versions[j].ID), f) + if err != nil { + log.Println("Failed to upload package: ", err) + os.Exit(1) + } + _, err = f.Seek(0, 0) + if err != nil { + log.Println("Failed to seek file: ", err) + os.Exit(1) + } + } + } + log.Printf("Uploaded to %d distros\n", totalDistros) +} + +func cleanup(config *Config) { + log.Println("Fetching distros") + distros, err := fetchDistros(config) if err != nil { - return err + log.Println("Failed to fetch distros: ", err) + os.Exit(1) } + destroyOlderVersions(config, "deb", distros.Deb) + destroyOlderVersions(config, "rpm", distros.Rpm) + log.Println("Cleanup completed") +} + +func destroyOlderVersions(config *Config, t string, distros []*Distro) { + totalDestroyed := 0 + for _, distro := range distros { + l := len(distro.Versions) + for j := l - 1; j >= 0; j-- { + versionGroups, err := fetchVersionGroups(config, t, distro.IndexName, distro.Versions[j].IndexName) + if err != nil { + log.Printf("Failed to fetch version groups for %s %s/%s: %s\n", t, distro.IndexName, distro.Versions[j].IndexName, err) + os.Exit(1) + } + noLongerSupported := l-j > config.MaxDistroVersionsToSupport + for _, versionGroup := range versionGroups { + versions, err := fetchVersions(config, versionGroup.VersionsURL) + if err != nil { + log.Printf("Failed to fetch versions for %s/%s: %s\n", distro.IndexName, distro.Versions[j].IndexName, err) + os.Exit(1) + } + destroyTo := len(versions) - config.MaxPackageVersionsToKeep + if noLongerSupported { + // If we no longer support this distro version, delete all versions + log.Printf("%s %s/%s is no longer supported, deleting all versions\n", t, distro.IndexName, distro.Versions[j].IndexName) + destroyTo = len(versions) + } + for i := 0; i < destroyTo; i++ { + v := versions[i] + log.Printf("Destroying version %s %s/%s: %s %s\n", t, distro.IndexName, distro.Versions[j].IndexName, v.Version, v.Filename) + err = deleteVersion(config, v.DestroyURL) + if err != nil { + log.Println("Failed to destroy version: ", err) + os.Exit(1) + } + totalDestroyed++ + } + } + } + } + log.Printf("%s: %d versions destroyed\n", t, totalDestroyed) +} + +func uploadPackage(config *Config, distroVersionId string, f *os.File) error { + var body bytes.Buffer writer := multipart.NewWriter(&body) part, err := writer.CreateFormFile("package[package_file]", f.Name()) if err != nil { @@ -80,14 +176,13 @@ func uploadPackage(config *Config, distroVersionId string, path string) error { if err != nil { return err } - req, err := http.NewRequest("POST", PackagecloudAPIURL+config.PackagecloudUser+"/"+config.PackagecloudRepo+"/packages.json", &body) + req, err := http.NewRequest("POST", PackagecloudAPIURL+"/api/v1/repos/"+config.PackagecloudUser+"/"+config.PackagecloudRepo+"/packages.json", &body) if err != nil { return err } req.Header.Set("Content-Type", writer.FormDataContentType()) req.SetBasicAuth(config.PackagecloudAPIKey, "") - client := &http.Client{} - resp, err := client.Do(req) + resp, err := http.DefaultClient.Do(req) if err != nil { return err } @@ -98,3 +193,107 @@ func uploadPackage(config *Config, distroVersionId string, path string) error { } return nil } + +type DistroVersion struct { + ID int `json:"id"` + IndexName string `json:"index_name"` +} + +type Distro struct { + IndexName string `json:"index_name"` + Versions []*DistroVersion +} + +type Distros struct { + Deb []*Distro `json:"deb"` + Rpm []*Distro `json:"rpm"` +} + +func fetchDistros(config *Config) (*Distros, error) { + req, err := http.NewRequest("GET", PackagecloudAPIURL+"/api/v1/distributions.json", nil) + if err != nil { + return nil, err + } + req.SetBasicAuth(config.PackagecloudAPIKey, "") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("unexpected status code: %d, body: %s", resp.StatusCode, string(b)) + } + distros := &Distros{} + json.NewDecoder(resp.Body).Decode(distros) + return distros, nil +} + +type PackageVersion struct { + CreatedAt time.Time `json:"created_at"` + DestroyURL string `json:"destroy_url"` + Version string `json:"version"` + Filename string `json:"filename"` +} + +func fetchVersions(config *Config, url string) ([]*PackageVersion, error) { + req, err := http.NewRequest("GET", PackagecloudAPIURL+url, nil) + if err != nil { + return nil, err + } + req.SetBasicAuth(config.PackagecloudAPIKey, "") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("unexpected status code: %d, body: %s", resp.StatusCode, string(b)) + } + versions := []*PackageVersion{} + json.NewDecoder(resp.Body).Decode(&versions) + return versions, nil +} + +type PackageVersionGroup struct { + VersionsURL string `json:"versions_url"` +} + +func fetchVersionGroups(config *Config, t string, distro string, name string) ([]*PackageVersionGroup, error) { + req, err := http.NewRequest("GET", PackagecloudAPIURL+"/api/v1/repos/"+config.PackagecloudUser+"/"+config.PackagecloudRepo+"/packages/"+t+"/"+distro+"/"+name+".json", nil) + if err != nil { + return nil, err + } + req.SetBasicAuth(config.PackagecloudAPIKey, "") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("unexpected status code: %d, body: %s", resp.StatusCode, string(b)) + } + versions := []*PackageVersionGroup{} + json.NewDecoder(resp.Body).Decode(&versions) + return versions, nil +} + +func deleteVersion(config *Config, url string) error { + req, err := http.NewRequest("DELETE", PackagecloudAPIURL+url, nil) + if err != nil { + return err + } + req.SetBasicAuth(config.PackagecloudAPIKey, "") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return fmt.Errorf("unexpected status code: %d, body: %s", resp.StatusCode, string(b)) + } + return nil +}