From 78239b8a599c83acbfaf3107b8576c27f2d9d68e Mon Sep 17 00:00:00 2001 From: Dominic Ricottone Date: Tue, 6 Jun 2023 18:59:41 -0500 Subject: [PATCH] Initial commit --- .gitignore | 2 + LICENSE.md | 31 ++++++++ Makefile | 13 ++++ README.md | 63 +++++++++++++++ apk.go | 146 +++++++++++++++++++++++++++++++++++ cli.go | 64 ++++++++++++++++ docker.go | 119 ++++++++++++++++++++++++++++ go.mod | 22 ++++++ main.go | 173 +++++++++++++++++++++++++++++++++++++++++ package.go | 20 +++++ resolver.go | 209 ++++++++++++++++++++++++++++++++++++++++++++++++++ rsync.go | 134 ++++++++++++++++++++++++++++++++ sourcetree.go | 52 +++++++++++++ 13 files changed, 1048 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE.md create mode 100644 Makefile create mode 100644 README.md create mode 100644 apk.go create mode 100644 cli.go create mode 100644 docker.go create mode 100644 go.mod create mode 100644 main.go create mode 100644 package.go create mode 100644 resolver.go create mode 100644 rsync.go create mode 100644 sourcetree.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bac6d8a --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +simple-builder +go.sum diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..41dc2f1 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,31 @@ +BSD 3-Clause License +==================== + +_Copyright (c) 2021, Dominic Ricottone_ +_All rights reserved._ + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..bee0d50 --- /dev/null +++ b/Makefile @@ -0,0 +1,13 @@ +simple-builder: + go get -u + go build . + +clean: + rm --force go.sum moby-demo + rm --force --recursive dir1 dir2 + +PWD=$(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +install: + ln -s $(PWD)simple-builder ~/.local/bin/simple-builder + +.PHONY: clean install diff --git a/README.md b/README.md new file mode 100644 index 0000000..e80655e --- /dev/null +++ b/README.md @@ -0,0 +1,63 @@ +# Simple Builder + +A simple package builder. +It leverages `docker(1)` for builds. +It leverages `rsync(1)` for file transfers. + +``` +simple-builder -repository /local/path/to/package/repository/alpine/v3.17/aarch64 +``` + +It autodetects architecture based on the repository, although an option for +explicit architecture is also available. + +``` +simple-builder -repository user:/var/pkgs -architecture amd64 +``` + +**Note: +Only supports ARM64 (a.k.a. AArch64 or ARM64v8) and AMD64 (a.k.a. x86_64) +currently.** + +It scans a package source directory for packages that could be built. +This defaults to `./src` but can be configured + +``` +simple-builder -repository rsync://user@example.com:8888 -architecture arm64 -source /usr/local/src +``` + +**Note: +The package source directory should be structured like `$src/$package/*`.** + +It parses package source files (i.e. `APKBUILD`s, etc.) for versioned packages +that should exist. +If a versioned package does not exist, it queues those package to be built. + +It also understands dependencies. +It will return an error if there is a circular dependency. +If a package has been built, it asserts that any other packages which depend +on the first one should be updated as well. +It will similarly return an error if a breaking build might be queued. + +Packages are built into a local folder. +This defaults to `./pkg` and but can be configured. + +``` +simple-builder -repository host:/var/alpine/v3.17/x86_64 -destination /var/alpine/v3.17/x86_64 +``` + +On success, both the package and APKINDEX files are pushed to the repository +immediately. + +It offers a simple command line interface. +Calling the binary without a command option will cause the program to print +summary information and exit. +Calling with the `-build` option will begin running through the build queue. +Use the `-verbose` flag to get diagnostic information. +Try `-help` for more information about all of this. + + +## License + +I license this work under the BSD 3-clause license. + diff --git a/apk.go b/apk.go new file mode 100644 index 0000000..d35ec3c --- /dev/null +++ b/apk.go @@ -0,0 +1,146 @@ +package main + +import ( + "bufio" + "fmt" + "os" + "path" + "regexp" + "strings" +) + +var ( + pattern_pkgver = regexp.MustCompile(`^pkgver=(.*)$`) + pattern_pkgrel = regexp.MustCompile(`^pkgrel=(.*)$`) + pattern_depends = regexp.MustCompile(`^depends="(.*)"$`) + pattern_apkname = regexp.MustCompile(`^([A-Za-z0-9._-]+)-([0-9]+\.[0-9]+(\.[0-9]+)?(\.[0-9]+)?-r[0-9]+)\.apk$`) +) + +// Scan a directory for an APKBUILD file. +func find_apkbuild(pkg *Package, directory string) error { + members, err := os.ReadDir(directory) + if (err != nil) { + return err + } + + for _, member := range members { + if (member.Name() == "APKBUILD") { + err = parse_apkbuild(pkg, path.Join(directory, member.Name())) + if (err != nil) { + return err + } + return nil + } + } + + return fmt.Errorf("No APKBUILD in %s", pkg.Name) +} + +// Parse an APKBUILD file. Given an existing Package, add core information +// (Version, Dependencies) as it is identified. +func parse_apkbuild(pkg *Package, filename string) error { + pkgver := "" + pkgrel := "" + depends := []string{} + inside_depends := false + + file, err := os.Open(filename) + if (err != nil) { + return err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + + match := pattern_pkgver.FindStringSubmatch(line) + if (match != nil) { + pkgver = match[1] + } + match = pattern_pkgrel.FindStringSubmatch(line) + if (match != nil) { + pkgrel = match[1] + } + + if (inside_depends == true) { + if (line == "\"") { + inside_depends = false + } else { + depends = append(depends, parse_list(line)...) + } + } else if (line == "depends=\"") { + inside_depends = true + } + + match = pattern_depends.FindStringSubmatch(line) + if (match != nil) { + depends = append(depends, parse_list(match[1])...) + } + } + + err = scanner.Err() + if (err != nil) { + return err + } + + if (pkgver == "") || (pkgrel == "") { + return fmt.Errorf("APKBUILD is incomplete in %s", pkg.Name) + } + pkg.Version = pkgver + "-r" + pkgrel + + if (len(depends) != 0) { + pkg.Dependencies = depends + } + + return nil +} + +// Scan a filename for an apk file. If one is identified, create a Package to +// represent it with all available information (Name and Version). +func find_apk(filename string) (Package, error) { + match := pattern_apkname.FindStringSubmatch(filename) + if (match != nil) { + return new_package_with_version(match[1], match[2]), nil + } + return Package{}, fmt.Errorf("Could not identify apk in %s", filename) +} + +// Construct the local directory expected to be built into. +func expected_apkdir(local_dir, arch string) string { + if (arch == "amd64") { + return path.Join(local_dir, "x86_64") + } else if (arch == "arm64") { + return path.Join(local_dir, "aarch64") + } + return local_dir +} + +// Construct the apk filename expected to correspond to a Package. +func expected_apk(pkg Package) string { + return fmt.Sprintf("%s-%s.apk", pkg.Name, pkg.Version) +} + +// Reconstruct the relevant parts of the APKBUILD files that were parsed. +func dump_apkbuilds(packages []Package, debug_prefix string) { + total := len(packages) + for i, p := range packages { + version := strings.SplitN(p.Version, "-r", 2) + fmt.Printf("DEBUG-APK:[%d/%d] %s\n", i + 1, total, p.Name) + fmt.Printf("DEBUG-APK:pkgver=%s\n", version[0]) + fmt.Printf("DEBUG-APK:pkgrel=%s\n", version[1]) + has_dependencies := (0 < len(p.Dependencies)) + print_if(has_dependencies, "DEBUG-APK:depends=\"") + for _, d := range p.Dependencies { + fmt.Printf("DEBUG-APK:\t%s\n", d) + } + print_if(has_dependencies, "DEBUG-APK:\"") + fmt.Println("DEBUG-APK:") + } +} + +// Parse a string as a whitespace-delimited list. +func parse_list(list string) []string { + return strings.Split(strings.TrimSpace(list), " ") +} + diff --git a/cli.go b/cli.go new file mode 100644 index 0000000..cf4df4f --- /dev/null +++ b/cli.go @@ -0,0 +1,64 @@ +package main + +import ( + "fmt" + "path/filepath" + "regexp" + "strings" +) + +// Clean up -source SRCDIR. +func clean_source(srcdir string) string { + src, err := filepath.Abs(srcdir) + if (err != nil) { + panic(err) + } + return src +} + +// Clean up -destination PKGDIR +func clean_destination(pkgdir string) string { + dest, err := filepath.Abs(pkgdir) + if (err != nil) { + panic(err) + } + return dest +} + +// Clean up -repository CONNECTION +func clean_repository(connection string) string { + repo := strings.TrimSpace(connection) + + // To obtain a directory listing, the connection string must end in a + // forward slash. + if (strings.HasSuffix(repo, "/") == false) { + repo += "/" + } + + pattern, err := regexp.Compile(`^(([A-Za-z0-9][A-Za-z0-9._-]*@)?([A-Za-z0-9._-]+):)?(/[A-Za-z0-9._-]+)+/$`) + if (err != nil) { + panic(err) + } + match := pattern.FindStringSubmatch(repo) + if (match == nil) || (match[4] == "") { + panic(fmt.Sprintf("Connection string %s seems invalid", repo)) + } + + return repo +} + +// Clean up -arch ARCH +func clean_architecture(arch, repo string) string { + if (arch == "amd64" ) || (arch == "arm64") { + return arch + } + + if (strings.Contains(repo, "x86_64")) { + return "amd64" + } else if (strings.Contains(repo, "aarch64")) { + return "arm64" + } + + panic("Not a valid architecture") +} + diff --git a/docker.go b/docker.go new file mode 100644 index 0000000..5181c65 --- /dev/null +++ b/docker.go @@ -0,0 +1,119 @@ +package main + +import ( + "bufio" + "context" + "errors" + "fmt" + "os" + "os/signal" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/mount" + "github.com/docker/docker/client" + specs "github.com/opencontainers/image-spec/specs-go/v1" +) + +// Create a container for building a package, start the build, and branch +// based on the result. +func build_package(pkg Package, srcdir, pkgdir, arch string) error { + ctx := context.Background() + + cli, err := client.NewClientWithOpts(client.FromEnv) + if (err != nil) { + return err + } + + conf := container.Config{ + Image: "registry.intra.dominic-ricottone.com/apkbuilder:latest", + Cmd: []string{pkg.Name}, + } + + con_conf := container.HostConfig{ + Mounts: []mount.Mount{ + { + Type: mount.TypeBind, + Source: srcdir, + Target: "/home/builder/src", + }, + { + Type: mount.TypeBind, + Source: pkgdir, + Target: "/home/builder/packages/src", + }, + }, + } + + plats := specs.Platform{ + Architecture: arch, + OS: "linux", + } + + con, err := cli.ContainerCreate(ctx, &conf, &con_conf, nil, &plats, "") + if (err != nil) { + return err + } + + start_opts := types.ContainerStartOptions{} + + cli.ContainerStart(ctx, con.ID, start_opts) + + err = check_result(cli, ctx, con.ID) + if (err != nil) { + return err + } + + rm_opts := types.ContainerRemoveOptions{ + Force: true, + } + + cli.ContainerRemove(ctx, con.ID, rm_opts) + + return nil +} + +// Get the result of a build. Blocks until the build is complete. +func check_result(cli *client.Client, ctx context.Context, id string) error { + statusC, errC := cli.ContainerWait(ctx, id, container.WaitConditionNotRunning) + + sigC := make(chan os.Signal) + signal.Notify(sigC, os.Interrupt) + + select { + case _ = <-sigC: + return errors.New("Build interrupted") + + case err := <-errC: + if (err != nil) { + return err + } + + case status := <-statusC: + if status.StatusCode != 0 { + dump_logs(cli, ctx, id) + return errors.New("Build failed") + } + } + + return nil +} + +// Dump logs from a build. +func dump_logs(cli *client.Client, ctx context.Context, id string) { + conf := types.ContainerLogsOptions{ + ShowStdout: true, + } + + out, err := cli.ContainerLogs(ctx, id, conf) + if err != nil { + panic(err) + } + defer out.Close() + + scanner := bufio.NewScanner(out) + for scanner.Scan() { + fmt.Println(scanner.Text()) + } +} + diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..dcfde32 --- /dev/null +++ b/go.mod @@ -0,0 +1,22 @@ +module git.dominic-ricottone.com/~dricottone/simple-builder + +go 1.20 + +require ( + github.com/docker/docker v24.0.2+incompatible + github.com/opencontainers/image-spec v1.0.2 +) + +require ( + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + golang.org/x/mod v0.10.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/tools v0.9.3 // indirect +) diff --git a/main.go b/main.go new file mode 100644 index 0000000..95c7e35 --- /dev/null +++ b/main.go @@ -0,0 +1,173 @@ +package main + +import ( + "flag" + "fmt" +) + +var ( + verbose = flag.Bool("verbose", false, "Show debugging messages") + build = flag.Bool("build", false, "Build packages") + summary = flag.Bool("summary", false, "Summarize packages to build") + source = flag.String("source", "./src", "Directory of package sources") + destination = flag.String("destination", "./pkg", "Directory of packages") + architecture = flag.String("architecture", "detected from repository", "architecture to build") + repository = flag.String("repository", "", "Connection string for the remote package repository") +) + +// Conditionally print a string. +func print_if(condition bool, str string) { + if (condition == true) { + fmt.Println(str) + } +} + +// Print debugging information if -verbose was passed to the program. +func debug(str string) { + print_if(*verbose, str) +} + +// Identify Packages in the package source directory. +func list_package_sources(local_dir string) ([]Package, error) { + packages, err := walk_package_sources(local_dir) + if (err != nil) { + return nil, err + } + + if (*verbose == true) { + dump_apkbuilds(packages, "DEBUG-MAIN") + } + + return packages, nil +} + +// Identify Packages in the repository. +func list_repository(remote_dir string) ([]Package, error) { + packages, err := fetch_repository_listing(remote_dir) + if (err != nil) { + return nil, err + } + + if (*verbose == true) { + dump_apkbuilds(packages, "DEBUG-MAIN") + } + + return packages, nil +} + +// Compare Packages between the package source directory and the repository. +func compare_lists(local_dir, remote_dir string) ([]Package, error) { + queue := []Package{} + had_errors := false + + package_sources, err := list_package_sources(local_dir) + if (err != nil) { + return nil, err + } + + repository, err := list_repository(remote_dir) + if (err != nil) { + return nil, err + } + + for i, _ := range package_sources { + err = find_builds(&package_sources[i], &repository) + if (err != nil) { + return nil, err + } + + if (package_sources[i].Build == true) { + queue = append(queue, package_sources[i]) + } + + if (package_sources[i].Error == true) { + print_if(!had_errors, "Warnings:") + had_errors = true + fmt.Printf("%s %s - %s\n", package_sources[i].Name, package_sources[i].Version, package_sources[i].Message) + } + } + + err = find_breaking_builds(&queue) + if (err != nil) { + return nil, err + } + + err = sort_queue(&queue) + if (err != nil) { + return nil, err + } + + return queue, nil +} + +// Sort the Package list in-place. +func sort_queue(packages *[]Package) error { + visited := []string{} + resolved := []Package{} + + for i, _ := range (*packages) { + err := resolve_dependencies(&resolved, packages, i, &visited) + if (err != nil) { + return err + } + } + + (*packages) = resolved + return nil +} + +// Build Packages. +func build_packages(packages []Package, source, destination, arch, repository string) error { + for _, pkg := range packages { + debug(fmt.Sprintf("Building %s...", pkg.Name)) + err := build_package(pkg, source, destination, arch) + if (err != nil) { + return err + } + + debug(fmt.Sprintf("Pushing %s...", pkg.Name)) + local_dir := expected_apkdir(destination, arch) + err = push_package(pkg, local_dir, repository) + if (err != nil) { + return err + } + } + + return nil +} + +// Print details about Packages queued for build. +func summarize_packages(packages []Package) { + if (len(packages) == 0) { + fmt.Println("Nothing to do") + } else { + fmt.Println("Packages to build:") + for _, p := range packages { + fmt.Printf(" %s %s - %s\n", p.Name, p.Version, p.Message) + } + fmt.Println("To start building, pass the `-build` option") + } +} + +func main() { + flag.Parse() + src := clean_source(*source) + pkg := clean_destination(*destination) + repo := clean_repository(*repository) + arch := clean_architecture(*architecture, repo) + + packages, err := compare_lists(src, repo) + if (err != nil) { + panic(err) + } + + if (*build == true) { + err = build_packages(packages, src, pkg, arch, repo) + if (err != nil) { + panic(err) + } + } else { + summarize_packages(packages) + } +} + diff --git a/package.go b/package.go new file mode 100644 index 0000000..8e26d5a --- /dev/null +++ b/package.go @@ -0,0 +1,20 @@ +package main + +// Package stores the core information about a software package. +type Package struct { + Name string + Version string + Dependencies []string + Message string + Build bool + Error bool +} + +func new_package(name string) Package { + return Package{name, "", []string{}, "", false, false} +} + +func new_package_with_version(name, version string) Package { + return Package{name, version, []string{}, "", false, false} +} + diff --git a/resolver.go b/resolver.go new file mode 100644 index 0000000..5081ddd --- /dev/null +++ b/resolver.go @@ -0,0 +1,209 @@ +package main + +import ( + "fmt" + "regexp" + "strconv" +) + +var ( + pattern_version = regexp.MustCompile(`^([0-9]+)\.([0-9]+)(\.([0-9]+))?(\.([0-9]+))?-r([0-9]+)$`) +) + +func find_string(haystack *[]string, needle string) int { + for i, s := range (*haystack) { + if (s == needle) { + return i + } + } + return -1 +} + +func find_package(packages *[]Package, name string) int { + for i, p := range (*packages) { + if (p.Name == name) { + return i + } + } + return -1 +} + +// Determine the resolution order for packages in a dependencies chain. +// +// This implementation is derivative of the algorithm described at +// https://www.electricmonk.nl/docs/dependency_resolving_algorithm/dependency_resolving_algorithm.html . +// This document is copyrighted by Ferry Boender (2008-2018) and was first +// published May 25, 2010. +func resolve_dependencies(resolved, pkgs *[]Package, index int, visited *[]string) error { + pkg := (*pkgs)[index] + + i := find_package(resolved, pkg.Name) + if (i != -1) { + return nil + } + + *visited = append(*visited, pkg.Name) + + for _, dep := range pkg.Dependencies { + // If dep is resolved, skip + i = find_package(resolved, dep) + if (i != -1) { + continue + } + + i = find_string(visited, dep) + if (i != -1) { + return fmt.Errorf("Circular dependencies in %s and %s", pkg.Name, dep) + } + + // If dep is not a known package, skip + i = find_package(pkgs, dep) + if (i == -1) { + continue + } + + err := resolve_dependencies(resolved, pkgs, i, visited) + if (err != nil) { + return err + } + } + + *resolved = append(*resolved, pkg) + return nil +} + +// Find packages to build. +func find_builds(pkg *Package, repository *[]Package) error { + i := find_package(repository, (*pkg).Name) + + // Package is new. + if (i == -1) { + (*pkg).Build = true + (*pkg).Message = "new" + return nil + } + + ver := (*repository)[i].Version + diff, err := compare_versions(ver, (*pkg).Version) + if (err != nil) { + return err + } + + // Package is newer in repository. Probably an issue. + if (diff == -1) { + (*pkg).Message = fmt.Sprintf("repository has newer %s", ver) + (*pkg).Error = true + return nil + } + + // Package already exists, nothing to do. + if (diff == 0) { + return nil + } + + // Package has an update. + (*pkg).Build = true + (*pkg).Message = fmt.Sprintf("update from %s", ver) + return nil +} + +// Find builds that may break other packages that depend on the rebuild. +func find_breaking_builds(pkgs *[]Package) error { + for _, pkg := range (*pkgs) { + if (pkg.Build == true) { + continue + } + + // If any dependency is marked for build, this package might break + for _, dep := range pkg.Dependencies { + i := find_package(pkgs, dep) + if (i != -1) && ((*pkgs)[i].Build == false) { + return fmt.Errorf("Package %s depends on updated/new %s but won't be rebuilt", pkg.Name, dep) + } + } + } + + return nil +} + +// Parse a version string. +func parse_version_string(version string) ([5]int, error) { + ver := [5]int{} + + match := pattern_version.FindStringSubmatch(version) + if (match == nil) { + return ver, fmt.Errorf("cannot parse %s", version) + } + + major, err := strconv.Atoi(match[1]) + if (err != nil) { + return ver, err + } + ver[0] = major + + minor, err := strconv.Atoi(match[2]) + if (err != nil) { + return ver, err + } + ver[1] = minor + + sub, err := strconv.Atoi(match[4]) + if (err != nil) && (match[4] != "") { + return ver, err + } + ver[2] = sub + + subsub, err := strconv.Atoi(match[6]) + if (err != nil) && (match[6] != "") { + return ver, err + } + ver[3] = subsub + + release, err := strconv.Atoi(match[7]) + if (err != nil) { + return ver, err + } + ver[4] = release + + debug(fmt.Sprintf("DEBUG-RESOLVER:%s -> %s", version, ver)) + + return ver, nil +} + +// Compare two version strings. +func compare_versions(remote, local string) (int, error) { + remote_ver, err := parse_version_string(remote) + if (err != nil) { + return 0, err + } + + local_ver, err := parse_version_string(local) + if (err != nil) { + return 0, err + } + + if (local_ver[0] == remote_ver[0]) { + if (local_ver[1] == remote_ver[1]) { + if (local_ver[2] == remote_ver[2]) { + if (local_ver[3] == remote_ver[3]) { + if (local_ver[4] == remote_ver[4]) { + return 0, nil + } else if (local_ver[4] < remote_ver[4]) { + return -1, nil + } + } else if (local_ver[3] < remote_ver[3]) { + return -1, nil + } + } else if (local_ver[2] < remote_ver[2]) { + return -1, nil + } + } else if (local_ver[1] < remote_ver[1]) { + return -1, nil + } + } else if (local_ver[0] < remote_ver[0]) { + return -1, nil + } + + return 1, nil +} + diff --git a/rsync.go b/rsync.go new file mode 100644 index 0000000..b607cdf --- /dev/null +++ b/rsync.go @@ -0,0 +1,134 @@ +package main + +import ( + "bufio" + "errors" + "fmt" + "io" + "os/exec" + "path" + "regexp" +) + +var ( + pattern_rsync_stdout = regexp.MustCompile(`^[drwx-]{10} *[0-9,]+ *[0-9]{4}/[0-9]{2}/[0-9]{2} *[0-9]{2}:[0-9]{2}:[0-9]{2} ([A-Za-z0-9._-]+) *$`) +) + +// Fetch a listing from a directory that is serving as a package repository. +func fetch_repository_listing(remote_dir string) ([]Package, error) { + debug(fmt.Sprintf("DEBUG-RSYNC:rsync --list-only %s", remote_dir)) + cmd := exec.Command("rsync", "--list-only", remote_dir) + stdout, err := cmd.StdoutPipe() + if (err != nil) { + return nil, err + } + err = cmd.Start() + if (err != nil) { + return nil, err + } + + pkgs, err := parse_rsync_stdout(stdout) + if (err != nil) { + return nil, err + } + + err = cmd.Wait() + if (err != nil) { + return nil, err + } + + if (len(pkgs) == 0) { + return nil, errors.New("No packages found") + } + + uniq := []Package{} + for _, pkg := range pkgs { + i := find_package(&uniq, pkg.Name) + if (i == -1) { + uniq = append(uniq, pkg) + } else { + diff, err := compare_versions(uniq[i].Version, pkg.Version) + if (err != nil) { + return nil, err + } + if (diff == 1) { + uniq[i].Version = pkg.Version + } + } + } + + return uniq, nil +} + +// Parse `rsync(1)` output. +func parse_rsync_stdout(stdout io.Reader) ([]Package, error) { + packages := []Package{} + + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + line := scanner.Text() + + pkgs, err := parse_rsync_line(line) + if (err != nil) { + return nil, err + } + + packages = append(packages, pkgs...) + } + + err := scanner.Err() + if (err != nil) { + return nil, err + } + + return packages, nil +} + +// Parse a line of `rsync(1)` output into 0 or 1 packages. +func parse_rsync_line(line string) ([]Package, error) { + debug(fmt.Sprintf("DEBUG-RSYNC:%s", line)) + + match := pattern_rsync_stdout.FindStringSubmatch(line) + if (match != nil) { + name := match[1] + + // Return an empty slice. Not an error, but also not a package. + if (name == ".") || (name == "APKINDEX.tar.gz") { + return []Package{}, nil + } + + // Repackage return value into slice. + pkg, err := find_apk(name) + if (err != nil) { + return nil, err + } else { + return []Package{pkg}, nil + } + } + + return nil, errors.New("Failed to parse line of rsync stdout") +} + +// Push a built package and an updated APKINDEX to a package repository. +func push_package(pkg Package, local_dir, remote_dir string) error { + local_name := path.Join(local_dir, expected_apk(pkg)) + + debug(fmt.Sprintf("DEBUG-RSYNC:rsync %s %s", local_name, remote_dir)) + cmd := exec.Command("rsync", local_name, remote_dir) + err := cmd.Run() + if (err != nil) { + return err + } + + local_name = path.Join(local_dir, "APKINDEX.tar.gz") + + debug(fmt.Sprintf("DEBUG-RSYNC:rsync %s %s", local_name, remote_dir)) + cmd = exec.Command("rsync", local_name, remote_dir) + err = cmd.Run() + if (err != nil) { + return err + } + + return nil +} + diff --git a/sourcetree.go b/sourcetree.go new file mode 100644 index 0000000..30947db --- /dev/null +++ b/sourcetree.go @@ -0,0 +1,52 @@ +package main + +import ( + "errors" + "fmt" + "os" + "path" +) + +// Walk a local directory to identify package sources. +// +// Package sources should be organized like: +// ``` +// root +// - package1 +// - APKBUILD +// - ... +// - ... +// ``` +// Any files not matching `APKBUILD` are ignored. +// Any directories not containing an `APKBUILD` file are ignored. +// Any files directly under the root are ignored. +func walk_package_sources(root string) ([]Package, error) { + packages := []Package{} + + members, err := os.ReadDir(root) + if (err != nil) { + return nil, err + } + + for _, member := range members { + if (member.IsDir() == true) { + name := member.Name() + pkg := new_package(name) + + err = find_apkbuild(&pkg, path.Join(root, name)) + if (err != nil) { + debug(fmt.Sprintf("DEBUG-PKGSRC:%s", err)) + } else { + packages = append(packages, pkg) + debug(fmt.Sprintf("DEBUG-PKGSRC:Package %s found", name)) + } + } + } + + if (len(packages) == 0) { + return nil, errors.New("No packages found") + } + + return packages, nil +} + -- 2.45.2