Skip to content

Commit 4be83ad

Browse files
jankaljankal
committed
feat: add concurrency
1 parent f1dce7c commit 4be83ad

2 files changed

Lines changed: 148 additions & 60 deletions

File tree

‎main.go‎

Lines changed: 147 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,13 @@ package main
22

33
import (
44
"encoding/json"
5+
"errors"
56
"fmt"
67
"log"
78
"net/http"
89
"os"
910
"strings"
11+
"sync"
1012
"time"
1113
)
1214

@@ -21,12 +23,35 @@ type PackageVersion struct {
2123
Time string `json:"time"`
2224
}
2325

26+
type PackageDetails struct {
27+
Data PackageDetailsResponse
28+
URL string
29+
}
2430
type PackageDetailsResponse struct {
2531
Package struct {
2632
Versions map[string]PackageVersion `json:"versions"`
2733
} `json:"package"`
2834
}
2935

36+
func (d *PackageDetailsResponse) GetLatestVersion() *PackageVersion {
37+
var latestVersion *PackageVersion
38+
var latestTime time.Time
39+
40+
for _, versionData := range d.Package.Versions {
41+
parsedTime, err := time.Parse(time.RFC3339, versionData.Time)
42+
if err != nil {
43+
log.Printf("Failed to parse time for package version: %s, error: %v", versionData.Time, err)
44+
continue
45+
}
46+
47+
if latestVersion == nil || parsedTime.After(latestTime) {
48+
latestTime = parsedTime
49+
latestVersion = &versionData
50+
}
51+
}
52+
return latestVersion
53+
}
54+
3055
type ShopwareExtensionMetadata struct {
3156
RepositoryUrl string `json:"repositoryUrl"`
3257
Ref string `json:"ref"`
@@ -96,6 +121,57 @@ func (c *GitHubClient) FetchLatestCommitTime(owner, repo string) (int, error) {
96121
return int(commitTime.Unix()), nil
97122
}
98123

124+
func (c *GitHubClient) GetMetadata(repositoryURL string) (bool, int) {
125+
parts := strings.Split(repositoryURL, "/")
126+
if len(parts) < 5 {
127+
log.Printf("Invalid GitHub URL: %s", repositoryURL)
128+
return false, 0
129+
}
130+
131+
owner := parts[3]
132+
repo := strings.TrimSuffix(parts[4], ".git")
133+
134+
resp, err := c.FetchFile(owner, repo, ".shopware-extension.yml")
135+
if err != nil {
136+
log.Printf("Failed to fetch .shopware-extension.yml file for %s/%s: %v", owner, repo, err)
137+
return false, 0
138+
}
139+
defer resp.Body.Close()
140+
141+
exists := resp.StatusCode == http.StatusOK
142+
if !exists && resp.StatusCode != http.StatusNotFound {
143+
log.Printf("Failed to fetch .shopware-extension.yml for %s/%s: %v", owner, repo, err)
144+
}
145+
146+
commitTime, err := c.FetchLatestCommitTime(owner, repo)
147+
if err != nil {
148+
log.Printf("Failed to fetch latest commit time for %s/%s: %v", owner, repo, err)
149+
}
150+
151+
return exists, commitTime
152+
}
153+
154+
func FetchPackageDetails(packageName string) (*PackageDetails, error) {
155+
detailsURL := fmt.Sprintf("https://packagist.org/packages/%s.json", packageName)
156+
detailsResp, err := http.Get(detailsURL)
157+
if err != nil {
158+
return nil, errors.New(fmt.Sprintf("Failed to fetch details for package %s: %v", packageName, err))
159+
}
160+
defer detailsResp.Body.Close()
161+
162+
var packageDetails PackageDetailsResponse
163+
if err := json.NewDecoder(detailsResp.Body).Decode(&packageDetails); err != nil {
164+
return nil, errors.New(fmt.Sprintf("Failed to decode details for package %s: %v", packageName, err))
165+
}
166+
167+
return &PackageDetails{
168+
Data: packageDetails,
169+
URL: detailsURL,
170+
}, nil
171+
}
172+
173+
var githubClient *GitHubClient
174+
99175
func main() {
100176
packageListURL := "https://packagist.org/packages/list.json?type=shopware-platform-plugin"
101177

@@ -111,56 +187,55 @@ func main() {
111187
log.Fatalf("Failed to decode package list: %v", err)
112188
}
113189

114-
packageData := make(map[string]*ShopwareExtensionMetadata)
115190
// Initialize the GitHub client
116-
githubClient := &GitHubClient{
191+
githubClient = &GitHubClient{
117192
HTTPClient: &http.Client{},
118193
Token: os.Getenv("GITHUB_TOKEN"),
119194
}
120-
// Iterate through each package to get the repository URL
121-
for _, packageName := range packageList.PackageNames {
122-
detailsURL := fmt.Sprintf("https://packagist.org/packages/%s.json", packageName)
123-
detailsResp, err := http.Get(detailsURL)
124-
if err != nil {
125-
log.Printf("Failed to fetch details for package %s: %v", packageName, err)
126-
continue
127-
}
128-
defer detailsResp.Body.Close()
129195

130-
var packageDetails PackageDetailsResponse
131-
if err := json.NewDecoder(detailsResp.Body).Decode(&packageDetails); err != nil {
132-
log.Printf("Failed to decode details for package %s: %v", packageName, err)
133-
continue
134-
}
196+
var wg sync.WaitGroup
135197

136-
var latestVersion *PackageVersion
137-
var latestTime time.Time
198+
workerCount := 10
199+
jobCount := len(packageList.PackageNames)
138200

139-
for _, versionData := range packageDetails.Package.Versions {
140-
parsedTime, err := time.Parse(time.RFC3339, versionData.Time)
141-
if err != nil {
142-
log.Printf("Failed to parse time for package version: %s, error: %v", versionData.Time, err)
143-
continue
144-
}
201+
jobs := make(chan Job, jobCount)
202+
results := make(chan Result, jobCount)
145203

146-
if latestVersion == nil || parsedTime.After(latestTime) {
147-
latestTime = parsedTime
148-
latestVersion = &versionData
149-
}
204+
wg.Add(workerCount)
205+
for w := 1; w <= workerCount; w++ {
206+
go worker(jobs, results, &wg)
207+
}
208+
209+
var resultsWg sync.WaitGroup
210+
resultsWg.Add(1)
211+
go collectResults(results, &resultsWg)
212+
213+
for _, packageName := range packageList.PackageNames {
214+
jobs <- Job{
215+
PackageName: packageName,
150216
}
217+
}
218+
close(jobs)
219+
wg.Wait()
220+
close(results)
221+
resultsWg.Wait()
222+
}
151223

152-
if latestVersion != nil {
153-
log.Printf("Package: %s, Repository: %s\n", packageName, latestVersion.Source.URL)
154-
if strings.Contains(latestVersion.Source.URL, "github.com") {
155-
extension, commitTime := checkShopwareExtensionFile(latestVersion.Source.URL, githubClient)
156-
if extension {
157-
packageData[packageName] = &ShopwareExtensionMetadata{
158-
RepositoryUrl: latestVersion.Source.URL,
159-
Ref: detailsURL,
160-
LatestCommitTime: commitTime,
161-
}
162-
}
163-
}
224+
type Job struct {
225+
PackageName string
226+
}
227+
228+
type Result struct {
229+
Data *PackageData
230+
}
231+
232+
func collectResults(results chan Result, wg *sync.WaitGroup) {
233+
defer wg.Done()
234+
235+
packageData := make(map[string]*ShopwareExtensionMetadata)
236+
for result := range results {
237+
if result.Data != nil {
238+
packageData[result.Data.PackageName] = result.Data.Metadata
164239
}
165240
}
166241

@@ -183,32 +258,45 @@ func main() {
183258
log.Println("Shopware extensions data written to shopware_extensions.json")
184259
}
185260

186-
func checkShopwareExtensionFile(repositoryURL string, client *GitHubClient) (bool, int) {
187-
parts := strings.Split(repositoryURL, "/")
188-
if len(parts) < 5 {
189-
log.Printf("Invalid GitHub URL: %s", repositoryURL)
190-
return false, 0
261+
func worker(jobs <-chan Job, results chan<- Result, wg *sync.WaitGroup) {
262+
defer wg.Done()
263+
for job := range jobs {
264+
results <- Result{
265+
Data: ProcessPackage(job.PackageName),
266+
}
191267
}
268+
}
192269

193-
owner := parts[3]
194-
repo := strings.TrimSuffix(parts[4], ".git")
270+
type PackageData struct {
271+
PackageName string
272+
Metadata *ShopwareExtensionMetadata
273+
}
195274

196-
resp, err := client.FetchFile(owner, repo, ".shopware-extension.yml")
275+
func ProcessPackage(packageName string) *PackageData {
276+
packageDetails, err := FetchPackageDetails(packageName)
197277
if err != nil {
198-
log.Printf("Failed to fetch .shopware-extension.yml file for %s/%s: %v", owner, repo, err)
199-
return false, 0
278+
log.Printf("%v", err)
279+
return nil
200280
}
201-
defer resp.Body.Close()
202281

203-
exists := resp.StatusCode == http.StatusOK
204-
if !exists && resp.StatusCode != http.StatusNotFound {
205-
log.Printf("Failed to fetch .shopware-extension.yml for %s/%s: %v", owner, repo, err)
206-
}
282+
latestVersion := packageDetails.Data.GetLatestVersion()
207283

208-
commitTime, err := client.FetchLatestCommitTime(owner, repo)
209-
if err != nil {
210-
log.Printf("Failed to fetch latest commit time for %s/%s: %v", owner, repo, err)
284+
if latestVersion != nil {
285+
log.Printf("Package: %s, Repository: %s\n", packageName, latestVersion.Source.URL)
286+
if strings.Contains(latestVersion.Source.URL, "github.com") {
287+
extension, commitTime := githubClient.GetMetadata(latestVersion.Source.URL)
288+
if extension {
289+
return &PackageData{
290+
PackageName: packageName,
291+
Metadata: &ShopwareExtensionMetadata{
292+
RepositoryUrl: latestVersion.Source.URL,
293+
Ref: packageDetails.URL,
294+
LatestCommitTime: commitTime,
295+
},
296+
}
297+
}
298+
}
211299
}
212300

213-
return exists, commitTime
301+
return nil
214302
}

‎shopware_extensions_v1.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -206,5 +206,5 @@
206206
"latestCommitTime": 1740863916
207207
}
208208
},
209-
"generatedAt": 1741475031
209+
"generatedAt": 1741476474
210210
}

0 commit comments

Comments
 (0)