diff --git a/.gitignore b/.gitignore index 0a764a4..9592128 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ env +debian +cache/* +!cache/.gitkeep diff --git a/Makefile b/Makefile index 80fcff5..1e5e401 100644 --- a/Makefile +++ b/Makefile @@ -73,8 +73,18 @@ nk.bin: ./nkbin_maker/bsd-ce ./u-boot-brain/u-boot.bin debian: + @if [ "$(shell uname)" != "Linux" ]; then \ + echo "Debootstrap is only available in Linux!"; \ + exit 1; \ + fi mkdir debian - sudo debootstrap --arch=armel --foreign buster debian/ + sudo debootstrap --arch=armel --foreign buster debian/ http://localhost:65432/debian/ sudo cp /usr/bin/qemu-arm-static debian/usr/bin/ - sudo cp setup_debian.sh debian/ + sudo cp ./tools/setup_debian.sh debian/ sudo chroot debian /setup_debian.sh + +.PHONY: +aptcache: + ./tools/aptcache_linux_amd64 \ + -rule 'local=localhost:65432, remote=ftp.jaist.ac.jp' \ + -rule 'local=localhost:65433, remote=security.debian.org' diff --git a/README.md b/README.md index 4b45708..10de12d 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,29 @@ Build Linux 1. Confirm that `linux-brain/arch/arm/boot/zImage` exists. +Bootstrap Debian 10 (buster) +---------------------------- + +1. Partition an SD card into two partitions. + + - 1st: FAT32 (vfat), about 100MB + - 2st: ext4, fill the remaining area + +1. Build and copy the Linux kernel. + + - Run `make ldefconfig lbuild`. + - Copy `/linux-brain/arch/arm/boot/zImage` and `/linux-brain/arch/arm/boot/dts/imx28-evk.dtb` into the 1st partition. + +1. Run APT cache in background (mandatory): `make aptcache`. + +1. Run `make debian`. + +1. Copy all contents in `./debian` into the 2nd partition. + + - `sudo cp -ar ./debian/* /path/to/your/sd/2nd/partition/` + - Please make sure that all attributes are preserved with `-a` flag. + + Watch changes in submodules & auto-build ---------------------------------------- diff --git a/cache/.gitkeep b/cache/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tools/aptcache.go b/tools/aptcache.go new file mode 100644 index 0000000..9fbc499 --- /dev/null +++ b/tools/aptcache.go @@ -0,0 +1,282 @@ +/* + +* See https://github.com/puhitaku/empera for the original source code. + +MIT License + +Copyright (c) 2020 Takumi Sueda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +*/ + +package main + +import ( + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "time" +) + +type Proxy struct { + remote string + + cli *http.Client + cache map[string]struct{} + cacheLock sync.Mutex +} + +func NewProxy() (*Proxy, error) { + p := &Proxy{ + cli: http.DefaultClient, + cache: map[string]struct{}{}, + } + + stat, err := os.Stat("cache") + if err != nil { + if err.(*os.PathError).Err != syscall.ENOENT { + return nil, fmt.Errorf("failed to stat cache directory: %s", err) + } + err = os.Mkdir("cache", 0755) + if err != nil { + return nil, fmt.Errorf("failed to create cache directory: %s", err) + } + } else if !stat.IsDir() { + return nil, fmt.Errorf("non-directory 'cache' exists") + } + + matches, err := filepath.Glob("cache/*") + if err != nil { + return nil, fmt.Errorf("failed to glob cache directory: %s", err) + } + + for i := range matches { + p.cache[strings.TrimPrefix(matches[i], "cache/")] = struct{}{} + } + return p, nil +} + +func (p *Proxy) Run(local, remote string) { + p.remote = remote + err := http.ListenAndServe(local, p) + if err != nil { + panic(err) + } +} + +// ServeHTTP implements http.Handler interface +func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var err error + encoded := url.PathEscape(r.URL.Path) + + exclude := []string{"Release", "Packages", "Contents"} + nocache := false + for _, ex := range exclude { + nocache = nocache || strings.Contains(encoded, ex) + } + + if nocache { + fmt.Printf("GET (no cache): %s%s -> ", p.remote, r.URL.Path) + err = p.fetchFromRemote(w, r, false) + } else if _, ok := p.cache[encoded]; ok { + fmt.Printf("GET (cache hit): %s%s -> ", p.remote, r.URL.Path) + err = p.fetchFromCache(w, r) + } else { + fmt.Printf("GET (cache miss): %s%s -> ", p.remote, r.URL.Path) + err = p.fetchFromRemote(w, r, true) + } + + if err != nil { + fmt.Printf("%s\n", err) + } else { + fmt.Printf("200\n") + } +} + +func (p *Proxy) fetchFromRemote(w http.ResponseWriter, r *http.Request, cache bool) error { + var f io.WriteCloser = NullWriter{} + var err error + + encoded := url.PathEscape(r.URL.Path) + fpath := filepath.Join("cache", encoded) + + newURL, err := url.Parse(r.URL.String()) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return fmt.Errorf("failed to parse URL: %s", err) + } + newURL.Scheme = "http" + newURL.Host = p.remote + + req, err := http.NewRequest(http.MethodGet, newURL.String(), nil) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return fmt.Errorf("failed to create a new request: %s", err) + } + + req.Header = r.Header + res, err := p.cli.Do(req) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return fmt.Errorf("failed to GET: %s", err) + } + defer res.Body.Close() + + _, err = os.Stat(fpath) + if err != nil { + if err.(*os.PathError).Err != syscall.ENOENT { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return fmt.Errorf("failed to stat cached file: %s", err) + } + } + + if cache && res.StatusCode == http.StatusOK { + f, err = os.Create(fpath) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return fmt.Errorf("failed to create file: %s", err) + } + defer f.Close() + } + + for k, vs := range res.Header { + for _, v := range vs { + w.Header().Add(k, v) + } + } + w.WriteHeader(res.StatusCode) + + _, err = io.Copy(w, io.TeeReader(res.Body, f)) + if err != nil { + return fmt.Errorf("failed to copy: %s", err) + } + + if res.StatusCode == http.StatusOK { + p.cacheLock.Lock() + defer p.cacheLock.Unlock() + p.cache[encoded] = struct{}{} + return nil + } else { + return fmt.Errorf(strconv.Itoa(res.StatusCode)) + } +} + +func (p *Proxy) fetchFromCache(w http.ResponseWriter, r *http.Request) error { + encoded := url.PathEscape(r.URL.Path) + f, err := os.Open(filepath.Join("cache", encoded)) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return fmt.Errorf("failed to open '%s': %s", encoded, err) + } + defer f.Close() + + _, err = io.Copy(w, f) + if err != nil { + return fmt.Errorf("failed to copy: %s", err) + } + return nil +} + +type NullWriter struct{} + +func (w NullWriter) Write(b []byte) (int, error) { + return len(b), nil +} + +func (w NullWriter) Close() error { + return nil +} + +type rule struct { + Local, Remote string +} + +type rules []rule + +func (r *rules) String() string { + return "" +} + +func (r *rules) Set(raw string) error { + var local, remote string + + kvs := strings.Split(raw, ",") + for _, kv := range kvs { + tokens := strings.Split(kv, "=") + if len(tokens) != 2 { + return fmt.Errorf("rule is malformed") + } + tokens[0], tokens[1] = strings.TrimSpace(tokens[0]), strings.TrimSpace(tokens[1]) + switch tokens[0] { + case "local": + local = tokens[1] + case "remote": + remote = tokens[1] + default: + return fmt.Errorf("rule has unknown key: '%s'", tokens[0]) + } + } + + if local == "" || remote == "" { + return fmt.Errorf("rule lacks mendatory keys: 'local' and/or 'remote'") + } + + *r = append(*r, rule{Local: local, Remote: remote}) + return nil +} + +func main() { + var rules rules + flag.Var(&rules, "rule", "Proxy rule. example: -rule 'local=localhost:8080, remote=super.slow.repository.example.com'") + flag.Parse() + + if len(rules) == 0 { + fmt.Fprintf(os.Stderr, "Fatal: specify one or more rules.\n") + flag.Usage() + os.Exit(1) + } + + for i, rule := range rules { + fmt.Printf("Proxy Rule %d: %s -> %s\n", i+1, rule.Local, rule.Remote) + + p, err := NewProxy() + if err != nil { + panic(err) + } + go p.Run(rule.Local, rule.Remote) + } + for { + time.Sleep(9999999999) + } +} diff --git a/tools/aptcache_linux_amd64 b/tools/aptcache_linux_amd64 new file mode 100755 index 0000000..523f81d Binary files /dev/null and b/tools/aptcache_linux_amd64 differ diff --git a/tools/build_aptcache.sh b/tools/build_aptcache.sh new file mode 100755 index 0000000..4ac79cc --- /dev/null +++ b/tools/build_aptcache.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o aptcache_linux_amd64 . +strip aptcache_linux_amd64 diff --git a/scripts/setup_debian.sh b/tools/setup_debian.sh similarity index 76% rename from scripts/setup_debian.sh rename to tools/setup_debian.sh index 510570b..bd48796 100755 --- a/scripts/setup_debian.sh +++ b/tools/setup_debian.sh @@ -4,13 +4,14 @@ TIMEZONE="Asia/Tokyo" /debootstrap/debootstrap --second-stage +# Use local APT cache during debootstrap cat < /etc/apt/sources.list -deb http://ftp.jaist.ac.jp/debian buster main contrib non-free -deb-src http://ftp.jaist.ac.jp/debian buster main contrib non-free -deb http://ftp.jaist.ac.jp/debian buster-updates main contrib non-free -deb-src http://ftp.jaist.ac.jp/debian buster-updates main contrib non-free -deb http://security.debian.org/debian-security buster/updates main contrib non-free -deb-src http://security.debian.org/debian-security buster/updates main contrib non-free +deb http://localhost:65432/debian buster main contrib non-free +deb-src http://localhost:65432/debian buster main contrib non-free +deb http://localhost:65432/debian buster-updates main contrib non-free +deb-src http://localhost:65432/debian buster-updates main contrib non-free +deb http://localhost:65433/debian-security buster/updates main contrib non-free +deb-src http://localhost:65433/debian-security buster/updates main contrib non-free EOF cat < /etc/apt/apt.conf.d/90-norecommend @@ -46,6 +47,16 @@ DEBIAN_FRONTEND=noninteractive \ bash tmux vim htop \ midori pcmanfm lxterminal xterm gnome-terminal fonts-noto-cjk \ dbus udev build-essential flex bison pkg-config autotools-dev libtool autoconf automake \ - python3 python3-dev python3-setuptools python3-wheel python3-pip \ + python3 python3-dev python3-setuptools python3-wheel python3-pip python3-smbus \ resolvconf net-tools ssh openssh-client avahi-daemon +# Get wild +cat < /etc/apt/sources.list +deb http://ftp.jaist.ac.jp/debian buster main contrib non-free +deb-src http://ftp.jaist.ac.jp/debian buster main contrib non-free +deb http://ftp.jaist.ac.jp/debian buster-updates main contrib non-free +deb-src http://ftp.jaist.ac.jp/debian buster-updates main contrib non-free +deb http://security.debian.org/debian-security buster/updates main contrib non-free +deb-src http://security.debian.org/debian-security buster/updates main contrib non-free +EOF +