Merge pull request #4 from brain-hackers/bootstrap-improvement

Improvements of deboostrapping

.gitignore

@@ -1 +1,4 @@
 env
+debian
+cache/*
+!cache/.gitkeep

Makefile

@@ -73,8 +73,18 @@ nk.bin:
 	./nkbin_maker/bsd-ce ./u-boot-brain/u-boot.bin
 
 debian:
+	@if [ "$(shell uname)" != "Linux" ]; then \
+		echo "Debootstrap is only available in Linux!"; \
+		exit 1; \
+	fi
 	mkdir debian
-	sudo debootstrap --arch=armel --foreign buster debian/
+	sudo debootstrap --arch=armel --foreign buster debian/ http://localhost:65432/debian/
 	sudo cp /usr/bin/qemu-arm-static debian/usr/bin/
-	sudo cp setup_debian.sh debian/
+	sudo cp ./tools/setup_debian.sh debian/
 	sudo chroot debian /setup_debian.sh
+
+.PHONY:
+aptcache:
+	./tools/aptcache_linux_amd64 \
+		-rule 'local=localhost:65432, remote=ftp.jaist.ac.jp' \
+		-rule 'local=localhost:65433, remote=security.debian.org'

README.md

@@ -75,6 +75,29 @@ Build Linux
 1. Confirm that `linux-brain/arch/arm/boot/zImage` exists.
 
 
+Bootstrap Debian 10 (buster)
+----------------------------
+
+1. Partition an SD card into two partitions.
+
+    - 1st: FAT32 (vfat), about 100MB
+    - 2st: ext4, fill the remaining area
+
+1. Build and copy the Linux kernel.
+
+    - Run `make ldefconfig lbuild`.
+    - Copy `/linux-brain/arch/arm/boot/zImage` and `/linux-brain/arch/arm/boot/dts/imx28-evk.dtb` into the 1st partition.
+
+1. Run APT cache in background (mandatory): `make aptcache`.
+
+1. Run `make debian`.
+
+1. Copy all contents in `./debian` into the 2nd partition.
+
+    - `sudo cp -ar ./debian/* /path/to/your/sd/2nd/partition/`
+    - Please make sure that all attributes are preserved with `-a` flag.
+
+
 Watch changes in submodules & auto-build
 ----------------------------------------
 

tools/aptcache.go

@@ -0,0 +1,282 @@
+/*
+
+* See https://github.com/puhitaku/empera for the original source code.
+
+MIT License
+
+Copyright (c) 2020 Takumi Sueda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+)
+
+type Proxy struct {
+	remote string
+
+	cli *http.Client
+	cache map[string]struct{}
+	cacheLock sync.Mutex
+}
+
+func NewProxy() (*Proxy, error) {
+	p := &Proxy{
+		cli: http.DefaultClient,
+		cache: map[string]struct{}{},
+	}
+
+	stat, err := os.Stat("cache")
+	if err != nil {
+		if err.(*os.PathError).Err != syscall.ENOENT {
+			return nil, fmt.Errorf("failed to stat cache directory: %s", err)
+		}
+		err = os.Mkdir("cache", 0755)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create cache directory: %s", err)
+		}
+	} else if !stat.IsDir() {
+		return nil, fmt.Errorf("non-directory 'cache' exists")
+	}
+
+	matches, err := filepath.Glob("cache/*")
+	if err != nil {
+		return nil, fmt.Errorf("failed to glob cache directory: %s", err)
+	}
+
+	for i := range matches {
+		p.cache[strings.TrimPrefix(matches[i], "cache/")] = struct{}{}
+	}
+	return p, nil
+}
+
+func (p *Proxy) Run(local, remote string) {
+	p.remote = remote
+	err := http.ListenAndServe(local, p)
+	if err != nil {
+		panic(err)
+	}
+}
+
+// ServeHTTP implements http.Handler interface
+func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	var err error
+	encoded := url.PathEscape(r.URL.Path)
+
+	exclude := []string{"Release", "Packages", "Contents"}
+	nocache := false
+	for _, ex := range exclude {
+		nocache = nocache || strings.Contains(encoded, ex)
+	}
+
+	if nocache {
+		fmt.Printf("GET (no cache): %s%s -> ", p.remote, r.URL.Path)
+		err = p.fetchFromRemote(w, r, false)
+	} else if _, ok := p.cache[encoded]; ok {
+		fmt.Printf("GET (cache hit): %s%s -> ", p.remote, r.URL.Path)
+		err = p.fetchFromCache(w, r)
+	} else {
+		fmt.Printf("GET (cache miss): %s%s -> ", p.remote, r.URL.Path)
+		err = p.fetchFromRemote(w, r, true)
+	}
+
+	if err != nil {
+		fmt.Printf("%s\n", err)
+	} else {
+		fmt.Printf("200\n")
+	}
+}
+
+func (p *Proxy) fetchFromRemote(w http.ResponseWriter, r *http.Request, cache bool) error {
+	var f io.WriteCloser = NullWriter{}
+	var err error
+
+	encoded := url.PathEscape(r.URL.Path)
+	fpath := filepath.Join("cache", encoded)
+
+	newURL, err := url.Parse(r.URL.String())
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return fmt.Errorf("failed to parse URL: %s", err)
+	}
+	newURL.Scheme = "http"
+	newURL.Host = p.remote
+
+	req, err := http.NewRequest(http.MethodGet, newURL.String(), nil)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return fmt.Errorf("failed to create a new request: %s", err)
+	}
+
+	req.Header = r.Header
+	res, err := p.cli.Do(req)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return fmt.Errorf("failed to GET: %s", err)
+	}
+	defer res.Body.Close()
+
+	_, err = os.Stat(fpath)
+	if err != nil {
+		if err.(*os.PathError).Err != syscall.ENOENT {
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(err.Error()))
+			return fmt.Errorf("failed to stat cached file: %s", err)
+		}
+	}
+
+	if cache && res.StatusCode == http.StatusOK {
+		f, err = os.Create(fpath)
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(err.Error()))
+			return fmt.Errorf("failed to create file: %s", err)
+		}
+		defer f.Close()
+	}
+
+	for k, vs := range res.Header {
+		for _, v := range vs {
+			w.Header().Add(k, v)
+		}
+	}
+	w.WriteHeader(res.StatusCode)
+
+	_, err = io.Copy(w, io.TeeReader(res.Body, f))
+	if err != nil {
+		return fmt.Errorf("failed to copy: %s", err)
+	}
+
+	if res.StatusCode == http.StatusOK {
+		p.cacheLock.Lock()
+		defer p.cacheLock.Unlock()
+		p.cache[encoded] = struct{}{}
+		return nil
+	} else {
+		return fmt.Errorf(strconv.Itoa(res.StatusCode))
+	}
+}
+
+func (p *Proxy) fetchFromCache(w http.ResponseWriter, r *http.Request) error {
+	encoded := url.PathEscape(r.URL.Path)
+	f, err := os.Open(filepath.Join("cache", encoded))
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return fmt.Errorf("failed to open '%s': %s", encoded, err)
+	}
+	defer f.Close()
+
+	_, err = io.Copy(w, f)
+	if err != nil {
+		return fmt.Errorf("failed to copy: %s", err)
+	}
+	return nil
+}
+
+type NullWriter struct{}
+
+func (w NullWriter) Write(b []byte) (int, error) {
+	return len(b), nil
+}
+
+func (w NullWriter) Close() error {
+	return nil
+}
+
+type rule struct {
+	Local, Remote string
+}
+
+type rules []rule
+
+func (r *rules) String() string {
+	return ""
+}
+
+func (r *rules) Set(raw string) error {
+	var local, remote string
+
+	kvs := strings.Split(raw, ",")
+	for _, kv := range kvs {
+		tokens := strings.Split(kv, "=")
+		if len(tokens) != 2 {
+			return fmt.Errorf("rule is malformed")
+		}
+		tokens[0], tokens[1] = strings.TrimSpace(tokens[0]), strings.TrimSpace(tokens[1])
+		switch tokens[0] {
+		case "local":
+			local = tokens[1]
+		case "remote":
+			remote = tokens[1]
+		default:
+			return fmt.Errorf("rule has unknown key: '%s'", tokens[0])
+		}
+	}
+
+	if local == "" || remote == "" {
+		return fmt.Errorf("rule lacks mendatory keys: 'local' and/or 'remote'")
+	}
+
+	*r = append(*r, rule{Local: local, Remote: remote})
+	return nil
+}
+
+func main() {
+	var rules rules
+	flag.Var(&rules, "rule", "Proxy rule. example: -rule 'local=localhost:8080, remote=super.slow.repository.example.com'")
+	flag.Parse()
+
+	if len(rules) == 0 {
+		fmt.Fprintf(os.Stderr, "Fatal: specify one or more rules.\n")
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	for i, rule := range rules {
+		fmt.Printf("Proxy Rule %d: %s -> %s\n", i+1, rule.Local, rule.Remote)
+
+		p, err := NewProxy()
+		if err != nil {
+			panic(err)
+		}
+		go p.Run(rule.Local, rule.Remote)
+	}
+	for {
+		time.Sleep(9999999999)
+	}
+}

tools/build_aptcache.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o aptcache_linux_amd64 .
+strip aptcache_linux_amd64

tools/setup_debian.sh

@@ -4,13 +4,14 @@ TIMEZONE="Asia/Tokyo"
 
 /debootstrap/debootstrap --second-stage
 
+# Use local APT cache during debootstrap
 cat <<EOF > /etc/apt/sources.list
-deb http://ftp.jaist.ac.jp/debian buster main contrib non-free
+deb http://localhost:65432/debian buster main contrib non-free
-deb-src http://ftp.jaist.ac.jp/debian buster main contrib non-free
+deb-src http://localhost:65432/debian buster main contrib non-free
-deb http://ftp.jaist.ac.jp/debian buster-updates main contrib non-free
+deb http://localhost:65432/debian buster-updates main contrib non-free
-deb-src http://ftp.jaist.ac.jp/debian buster-updates main contrib non-free
+deb-src http://localhost:65432/debian buster-updates main contrib non-free
-deb http://security.debian.org/debian-security buster/updates main contrib non-free
+deb http://localhost:65433/debian-security buster/updates main contrib non-free
-deb-src http://security.debian.org/debian-security buster/updates main contrib non-free
+deb-src http://localhost:65433/debian-security buster/updates main contrib non-free
 EOF
 
 cat <<EOF > /etc/apt/apt.conf.d/90-norecommend
@@ -46,6 +47,16 @@ DEBIAN_FRONTEND=noninteractive \
                    bash tmux vim htop \
                    midori pcmanfm lxterminal xterm gnome-terminal fonts-noto-cjk \
                    dbus udev build-essential flex bison pkg-config autotools-dev libtool autoconf automake \
-                   python3 python3-dev python3-setuptools python3-wheel python3-pip \
+                   python3 python3-dev python3-setuptools python3-wheel python3-pip python3-smbus \
                    resolvconf net-tools ssh openssh-client avahi-daemon
 
+# Get wild
+cat <<EOF > /etc/apt/sources.list
+deb http://ftp.jaist.ac.jp/debian buster main contrib non-free
+deb-src http://ftp.jaist.ac.jp/debian buster main contrib non-free
+deb http://ftp.jaist.ac.jp/debian buster-updates main contrib non-free
+deb-src http://ftp.jaist.ac.jp/debian buster-updates main contrib non-free
+deb http://security.debian.org/debian-security buster/updates main contrib non-free
+deb-src http://security.debian.org/debian-security buster/updates main contrib non-free
+EOF
+