Merge branch 'dev'

Changelog.txt

@@ -25,7 +25,38 @@ Add tcp/udp --redir
 --- Mon Apr 23 2018
 Add --offline and --resize option by @NoobieDog
 
+--- Thu May 31 2018
+Correct --resize
+Fix bug with trying to sync big files
+Migrate from clize to docopt
+Refactor utils functions
+Refactor filesystem, config, options functions
+Disable audio to gain boot speed
+Add colors
+Add -add-qemu-options
+Add show command
+
+
 TODO:
+Add a new features sections
+	autostart
+	resize
+	show
+	https
+	full support of ext4
+	Package manager is preinstalled
+
+	we can share drive like this:
+arm_now start x86-core2 --add-qemu-options "-machine usb=on -usb -usbdevice disk:empty-ext3-2GB.img"
+
+find binary-samples/ -type f |tee filelist | e2cp -apv -d arm_now/rootfs.ext2:/
+10.0.2.2 is the host ip
+use mkisofs or genisoimage to create an iso from a folder. 
+https://unix.stackexchange.com/questions/90793/create-iso-image-from-folder-via-terminal-commands
+sudo mount -o loop output_image.iso iso/
+
+Add a wiki page describing every options, --redir ..  and use of alias
+Unit test
 For --offline option check if local filesystem are avaiable otherwise exit cleanly and print a error message.
 For --resize option check with a regex if the parameter is correct to avoid command/parameter injection.
 Search for new kernel and rootfs images, maybe I missed some of them that have ramfs inside the kernel image.

README.md

@@ -7,7 +7,7 @@ arm_now is a qemu powered tool that allows instant setup of virtual machines on
 
 # Install
 ```sh
-# pip3 install https://github.com/nongiach/arm_now/archive/master.zip
+# pip3 install https://github.com/nongiach/arm_now/archive/master.zip --upgrade
 ```
 
 # Start an arm Virtual Machine

arm_now/config.py

@@ -0,0 +1,61 @@
+class Config:
+    DIR = "arm_now/"
+    kernel = "kernel"
+    dtb = "dtb"
+    rootfs = "rootfs.ext2"
+    arch = "arch"
+    KERNEL = DIR + "kernel"
+    DTB = DIR + "dtb"
+    ROOTFS = DIR + "rootfs.ext2"
+    ARCH = DIR + "arch"
+    DOWNLOAD_CACHE_DIR = "/tmp/arm_now"
+
+qemu_options = {
+	"aarch64": ["aarch64", "-M virt -cpu cortex-a57 -smp 1 -kernel {kernel} -append 'root=/dev/vda console=ttyAMA0' -netdev user,id=eth0 -device virtio-net-device,netdev=eth0 -drive file={rootfs},if=none,format=raw,id=hd0 -device virtio-blk-device,drive=hd0"],
+        "armv5-eabi": ["arm", "-M vexpress-a9 -kernel {kernel} -sd {rootfs} -append 'root=/dev/mmcblk0 console=ttyAMA0 rw physmap.enabled=0 noapic'"], # check log
+        "armv6-eabihf": ["arm", "-M vexpress-a9 -kernel {kernel} -sd {rootfs} -append 'root=/dev/mmcblk0 console=ttyAMA0 rw physmap.enabled=0 noapic'"], # check log
+        "armv7-eabihf": ["arm", "-M vexpress-a9 -kernel {kernel} -sd {rootfs} -append 'root=/dev/mmcblk0 console=ttyAMA0 rw physmap.enabled=0 noapic'"], # check log
+        # "bfin":, TODO
+        # "m68k-68xxx":, TODO 
+        "m68k-coldfire": ["m68k", "-kernel {kernel} -hda {rootfs} -append 'root=/dev/sda console=ttyS0 rw physmap.enabled=0 noapic'"], # check log
+        "microblazebe": ["microblaze", "-M petalogix-s3adsp1800 -kernel {kernel} -nographic"], # rootfs is inside the kernel file, but we also have a separated rootfs if needed
+        "microblazeel": ["microblazeel", "-kernel {kernel} -hda {rootfs} -append 'root=/dev/sda console=tty0 rw physmap.enabled=0 noapic'"], # check log
+        "mips32": ["mips", "-kernel {kernel} -hda {rootfs} -append 'root=/dev/hda console=ttyS0 rw physmap.enabled=0 noapic'"], # check log
+        "mips32el": ["mipsel", "-kernel {kernel} -hda {rootfs} -append 'root=/dev/hda console=ttyS0 rw physmap.enabled=0 noapic'"], # check log
+	"mips32r5el": ["mipsel", "-machine malta -cpu P5600 -kernel {kernel} -drive file={rootfs},format=raw -append 'root=/dev/hda rw'"],
+        "mips32r6el": ["mipsel", "-M malta -cpu mips32r6-generic -kernel {kernel} -drive file={rootfs},format=raw -append root=/dev/hda -net nic,model=pcnet -net user"],
+        "mips64-n32": ["mips64", "-kernel {kernel} -hda {rootfs} -append 'root=/dev/hda console=ttyS0 rw physmap.enabled=0 noapic'"], # check log
+        "mips64el-n32": ["mips64el", "-kernel {kernel} -hda {rootfs} -append 'root=/dev/hda console=ttyS0 rw physmap.enabled=0 noapic'"], # check log
+        # "mips64r6el-n32":, TODO check log
+        # "mips64r6el-n32": ["mips64el", "-machine malta -kernel {kernel} -drive file={rootfs},format=raw -append 'root=/dev/hda rw console=ttyS0,'"], # check log
+        "nios2": ["nios2", "-kernel {kernel} -hda {rootfs} -append 'root=/dev/sda console=ttyS0 rw physmap.enabled=0 noapic'"], # check log
+        "powerpc64-e5500": ["ppc64", "-kernel {kernel} -hda {rootfs} -append 'root=/dev/sda console=ttyS0 rw physmap.enabled=0 noapic'"], # check log
+        "powerpc64-power8": ["ppc64", "-kernel {kernel} -hda {rootfs} -append 'root=/dev/sda console=ttyS0 rw physmap.enabled=0 noapic'"], # check log
+        "powerpc64le-power8": ["ppc64", "-kernel {kernel} -hda {rootfs} -append 'root=/dev/sda console=ttyS0 rw physmap.enabled=0 noapic'"], # check log
+        "sh-sh4": ["sh4", "-M r2d -serial vc -kernel {kernel} -hda {rootfs} -append 'root=/dev/sda console=ttyS0 rw physmap.enabled=0 noapic'"], # check log
+        # "sparc64":, TODO check log
+        # "sparc64": ["sparc64", "-M sun4u -kernel {kernel} -append 'root=/dev/sda console=ttyS0,115200' -drive file={rootfs},format=raw -net nic,model=e1000 -net user"], # this causes kernel crash
+        # ":sparcv8":, TODO, check log, 
+        # "sparcv8": ["sparc", "-machine SS-10 -kernel {kernel} -drive file={rootfs},format=raw -append 'root=/dev/sda console=ttyS0,115200' -net nic,model=lance -net user"], # error
+        # "x86-64-core-i7":["x86_64", "-kernel {kernel} -hda {rootfs} -append 'root=/dev/sda console=ttyS0 rw physmap.enabled=0 noapic'"], # old
+        "x86-64-core-i7" : ["x86_64", "-M pc -kernel {kernel} -drive file={rootfs},if=virtio,format=raw -append 'root=/dev/vda rw console=ttyS0' -net nic,model=virtio -net user"],
+        # "x86-core2" : ["i386", "-kernel {kernel} -hda {rootfs} -append 'root=/dev/sda console=ttyS0 rw physmap.enabled=0 noapic -net nic,model=virtio -net user'"],
+	"x86-core2": ["i386", "-M pc -kernel {kernel} -drive file={rootfs},if=virtio,format=raw -append 'root=/dev/vda rw console=ttyS0' -net nic,model=virtio -net user"], # fix opkg
+        "x86-i686":["i386", "-M pc -kernel {kernel} -drive file={rootfs},if=virtio,format=raw -append 'root=/dev/vda rw console=ttyS0' -net nic,model=virtio -net user"],
+        "xtensa-lx60": ["xtensa", "-M lx60 -cpu dc233c -monitor null -nographic -kernel {kernel} -monitor null"]
+        }
+
+""" The final user is not vulnerable to MITM attack, thoose http links are used to preconfigure the package manager.
+When you do a arm_now start armv5-eabi the image you are downloading already has opkg installed and
+everything is configured to use https, zero request are made over http.
+thx lucasduffey
+"""
+
+install_opkg = {
+        "armv5-eabi":"""wget -O - http://pkg.entware.net/binaries/armv5/installer/entware_install.sh | /bin/sh""",
+        "armv7-eabihf":"""wget -O - http://pkg.entware.net/binaries/armv5/installer/entware_install.sh | /bin/sh""",
+        "mips32el":"""wget -O - http://pkg.entware.net/binaries/mipsel/installer/installer.sh | /bin/sh""",
+        "x86-64-core-i7":"""wget -O - http://pkg.entware.net/binaries/x86-64/installer/entware_install.sh | /bin/sh""",
+        "x86-core2":"""wget -O - http://pkg.entware.net/binaries/x86-32/installer/entware_install.sh | /bin/sh""",
+        "x86-i686":"""wget -O - http://pkg.entware.net/binaries/x86-32/installer/entware_install.sh | /bin/sh""",
+}

arm_now/download.py

@@ -0,0 +1,112 @@
+import os
+import sys
+import re
+import requests
+import shutil
+from subprocess import check_call
+from collections import defaultdict
+from pathlib import Path
+
+# Exall is an exception manager based on decorator/context/callback
+# Check it out: https://github.com/nongiach/exall
+from exall import exall, ignore, print_warning, print_traceback, print_error
+from pySmartDL import SmartDL
+from .config import Config
+
+@exall(os.mkdir, FileExistsError, ignore)
+def download(url, filename, cache_directory):
+    filename_cache = url.split('/')[-1]
+    filename_cache = ''.join([ c for c in filename_cache if c.isdigit() or c.isalpha() ])
+    filename_cache = cache_directory + "/" + filename_cache
+    if os.path.exists(filename):
+        return
+    elif os.path.exists(filename_cache):
+        print("Already downloaded")
+        shutil.copyfile(filename_cache, filename)
+    else:
+        print("\nDownloading {} from {}".format(filename, url))
+        os.mkdir(cache_directory)
+        # wget.download(url, out=filename_cache)
+        obj = SmartDL(url, filename_cache)
+        obj.start()
+        shutil.copyfile(filename_cache, filename)
+
+def scrawl_kernel(arch):
+    re_href = re.compile('href="?({arch}[^ <>"]*)"?'.format(arch=arch))
+    url = "https://toolchains.bootlin.com/downloads/releases/toolchains/{arch}/test-system/".format(arch=arch)
+    response = requests.get(url + "?C=M;O=D")
+    text = response.text
+    links = re_href.findall(text)
+    links_dict = defaultdict(lambda : defaultdict(dict))
+    for link in links:
+        version = get_link_version(link)
+        libc = get_link_libc(link)
+        filetype = get_link_filetype(link)
+
+        # TODO: make sure they have been compiled at the same time
+        if filetype not in links_dict[version][libc]:
+            if filetype is None:
+                return None, None, None
+            links_dict[version][libc][filetype] = url + link
+    state = "stable" if "stable" in links_dict else "bleeding-edge"
+    libc = "uclibc" if "uclibc" in links_dict[state] else None
+    libc = "musl" if "musl" in links_dict[state] else libc
+    libc = "glibc" if "glibc" in links_dict[state] else libc
+    dtb = None if "dtb" not in links_dict[state][libc] else links_dict[state][libc]["dtb"]
+    rootfs = None if "rootfs" not in links_dict[state][libc] else links_dict[state][libc]["rootfs"]
+    kernel = None if "kernel" not in links_dict[state][libc] else links_dict[state][libc]["kernel"]
+    return kernel, dtb, rootfs
+
+def indexof_parse(url):
+    re_href = re.compile('\[DIR\].*href="?([^ <>"]*)"?')
+    response = requests.get(url)
+    text = response.text
+    links = re_href.findall(text)
+    return links
+
+libc = ["uclibc", "glibc", "musl"]
+def get_link_libc(link):
+    for i_libc in libc:
+        if i_libc in link:
+            return i_libc
+    return None
+
+def get_link_version(link):
+    if "bleeding-edge" in link:
+        return "bleeding-edge"
+    else:
+        return "stable"
+
+def get_link_filetype(link):
+    if ".cpio" in link or ".ext2" in link or "rootfs" in link:
+        return "rootfs"
+    elif "dtb" in link:
+        return "dtb"
+    elif "Image" in link or "vmlinux" in link or "linux.bin" in link:
+        return "kernel"
+    print("ERROR: I don't know this kind of file {}".format(link), file=sys.stderr)
+    # os.kill(0, 9)
+    return None
+
+@exall(os.mkdir, FileExistsError, ignore)
+def download_from_github(arch):
+    templates = str(Path.home()) + "/.config/arm_now/templates/"
+    os.makedirs(templates)
+    filename = arch + ".tar.xz"
+    URL = "https://github.com/nongiach/arm_now_templates/raw/master/"
+    download(URL + filename, templates + filename, Config.DOWNLOAD_CACHE_DIR)
+
+def download_image(arch, *, dest, real_source):
+    if real_source:
+        kernel, dtb, rootfs = scrawl_kernel(arch)
+        if kernel is None or rootfs is None:
+            pred("ERROR: couldn't download files for this arch", file=sys.stderr)
+            sys.exit(1)
+        download(kernel, dest + Config.kernel, Config.DOWNLOAD_CACHE_DIR)
+        if dtb: download(dtb, dest + Config.dtb, Config.DOWNLOAD_CACHE_DIR)
+        download(rootfs, dest + Config.rootfs, Config.DOWNLOAD_CACHE_DIR)
+    else:
+        download_from_github(arch)
+        check_call("tar xf ~/.config/arm_now/templates/{}.tar.xz {}".format(arch, dest), shell=True)
+    with open(dest + "/arch", "w") as F:
+        F.write(arch)