Browse Source

Merge isohybrid build support

K. Lange 2 years ago
parent
commit
0ed0f2d96d
5 changed files with 380 additions and 40 deletions
  1. 7 2
      .gitignore
  2. 32 37
      Makefile
  3. 1 1
      README.md
  4. 30 0
      util/mkdisk.sh
  5. 310 0
      util/update-extents.py

+ 7 - 2
.gitignore

@@ -8,8 +8,13 @@ cdrom/kernel
 cdrom/netinit
 cdrom/mod/*
 cdrom/ramdisk.img
-cdrom/boot/boot.sys
-cdrom/boot/efi.img
+cdrom/boot.sys
+cdrom/fat.img
+fatbase/kernel
+fatbase/netinit
+fatbase/mod/*
+fatbase/ramdisk.img
+fatbase/efi/boot/bootia32.efi
 *.efi
 util/tarballs
 util/build

+ 32 - 37
Makefile

@@ -64,7 +64,7 @@ KERNEL_ASMOBJS = $(filter-out kernel/symbols.o,$(patsubst %.S,%.o,$(wildcard ker
 
 # Kernel
 
-cdrom/kernel: ${KERNEL_ASMOBJS} ${KERNEL_OBJS} kernel/symbols.o
+fatbase/kernel: ${KERNEL_ASMOBJS} ${KERNEL_OBJS} kernel/symbols.o
 	${KCC} -T kernel/link.ld ${KCFLAGS} -nostdlib -o $@ ${KERNEL_ASMOBJS} ${KERNEL_OBJS} kernel/symbols.o -lgcc
 
 kernel/symbols.o: ${KERNEL_ASMOBJS} ${KERNEL_OBJS} util/generate_symbols.py
@@ -84,14 +84,14 @@ kernel/%.o: kernel/%.c ${HEADERS}
 
 # Modules
 
-cdrom/mod:
+fatbase/mod:
 	@mkdir -p $@
 
-MODULES = $(patsubst modules/%.c,cdrom/mod/%.ko,$(wildcard modules/*.c))
+MODULES = $(patsubst modules/%.c,fatbase/mod/%.ko,$(wildcard modules/*.c))
 
 HEADERS = $(shell find base/usr/include/kernel -type f -name '*.h')
 
-cdrom/mod/%.ko: modules/%.c ${HEADERS} | cdrom/mod
+fatbase/mod/%.ko: modules/%.c ${HEADERS} | fatbase/mod
 	${KCC} -T modules/link.ld -nostdlib ${KCFLAGS} -c -o $@ $<
 
 modules: ${MODULES}
@@ -99,20 +99,20 @@ modules: ${MODULES}
 # Root Filesystem
 
 base/dev:
-	mkdir -p base/dev
+	mkdir -p $@
 base/tmp:
-	mkdir -p base/tmp
+	mkdir -p $2
 base/proc:
-	mkdir -p base/proc
+	mkdir -p $@
 base/bin:
-	mkdir -p base/bin
+	mkdir -p $@
 base/lib:
-	mkdir -p base/lib
-cdrom/boot:
-	mkdir -p cdrom/boot
+	mkdir -p $@
+fatbase/efi/boot:
+	mkdir -p $@
 .make:
 	mkdir -p .make
-dirs: base/dev base/tmp base/proc base/bin base/lib cdrom/boot .make
+dirs: base/dev base/tmp base/proc base/bin base/lib fatbase/efi/boot .make
 
 # C Library
 
@@ -154,7 +154,7 @@ endif
 base/bin/init: apps/init.c base/lib/libc.a | dirs
 	$(CC) -static -Wl,-static $(CFLAGS) -o $@ $<
 
-cdrom/netinit: util/netinit.c base/lib/libc.a | dirs
+fatbase/netinit: util/netinit.c base/lib/libc.a | dirs
 	$(CC) -s -static -Wl,-static $(CFLAGS) -o $@ $<
 
 # Userspace applications
@@ -168,35 +168,30 @@ endif
 
 # Ramdisk
 
-cdrom/ramdisk.img: ${APPS_X} ${LIBS_X} base/lib/ld.so base/lib/libm.so $(shell find base) Makefile | dirs
-	genext2fs -B 4096 -d base -D util/devtable -U -b `util/calc-size.sh` -N 2048 cdrom/ramdisk.img
+fatbase/ramdisk.img: ${APPS_X} ${LIBS_X} base/lib/ld.so base/lib/libm.so $(shell find base) Makefile | dirs
+	genext2fs -B 4096 -d base -D util/devtable -U -b `util/calc-size.sh` -N 2048 $@
 
 # CD image
 
 ifeq (,$(wildcard /usr/lib32/crt0-efi-ia32.o))
-    EFI_XORRISO=
-    EFI_BOOT=
 else
-    EFI_XORRISO=-eltorito-alt-boot -e boot/efi.img -no-emul-boot -isohybrid-gpt-basdat
-    EFI_BOOT=cdrom/boot/efi.img
 endif
 
+EFI_XORRISO=-eltorito-alt-boot -e fat.img -no-emul-boot -isohybrid-gpt-basdat
+EFI_BOOT=cdrom/fat.img
+EFI_UPDATE=python util/update-extents.py
 
-image.iso: cdrom/ramdisk.img cdrom/boot/boot.sys cdrom/kernel cdrom/netinit ${MODULES}
-	xorriso -as mkisofs -R -J -c boot/bootcat \
-	  -b boot/boot.sys -no-emul-boot -boot-load-size 24 \
+image.iso: ${EFI_BOOT} cdrom/boot.sys fatbase/netinit ${MODULES}
+	xorriso -as mkisofs -R -J -c bootcat \
+	  -b boot.sys -no-emul-boot -boot-load-size 24 \
 	  ${EFI_XORRISO} \
 	  -o image.iso cdrom
+	${EFI_UPDATE}
 
 # Boot loader
 
-cdrom/boot/efi.img: boot/boot32.efi
-	-rm -f $@
-	fallocate -l 32M $@
-	mkfs.fat $@
-	-mmd  -i $@ '/EFI'
-	-mmd  -i $@ '/EFI/BOOT'
-	mcopy -i $@ boot/boot32.efi '::/EFI/BOOT/BOOTIA32.EFI'
+cdrom/fat.img: fatbase/ramdisk.img ${MODULES} fatbase/kernel fatbase/netinit fatbase/efi/boot/bootia32.efi
+	util/mkdisk.sh $@ fatbase
 
 EFI_CFLAGS=-fno-stack-protector -fpic -DEFI_FUNCTION_WRAPPER -m32 -DEFI_PLATFORM -ffreestanding -fshort-wchar -I /usr/include/efi -I /usr/include/efi/ia32 -mno-red-zone
 EFI_SECTIONS=-j .text -j .sdata -j .data -j .dynamic -j .dynsym -j .rel -j .rela -j .reloc
@@ -205,10 +200,10 @@ boot/efi.so: boot/cstuff.c boot/*.h
 	$(CC) ${EFI_CFLAGS} -c -o boot/efi.o $<
 	$(LD) boot/efi.o /usr/lib32/crt0-efi-ia32.o -nostdlib -znocombreloc -T /usr/lib32/elf_ia32_efi.lds -shared -Bsymbolic -L /usr/lib32 -lefi -lgnuefi -o boot/efi.so
 
-boot/boot32.efi: boot/efi.so
-	objcopy ${EFI_SECTIONS} --target=efi-app-ia32 boot/efi.so boot/boot32.efi
+fatbase/efi/boot/bootia32.efi: boot/efi.so
+	objcopy ${EFI_SECTIONS} --target=efi-app-ia32 boot/efi.so $@
 
-cdrom/boot/boot.sys: boot/boot.o boot/cstuff.o boot/link.ld ${EFI_BOOT} | cdrom/boot
+cdrom/boot.sys: boot/boot.o boot/cstuff.o boot/link.ld | dirs
 	${KLD} -T boot/link.ld -o $@ boot/boot.o boot/cstuff.o
 
 boot/cstuff.o: boot/cstuff.c boot/*.h
@@ -224,14 +219,14 @@ clean:
 	rm -f ${APPS_X}
 	rm -f libc/*.o libc/*/*.o
 	rm -f image.iso
-	rm -f cdrom/ramdisk.img
-	rm -f cdrom/boot/boot.sys
+	rm -f fatbase/ramdisk.img
+	rm -f cdrom/boot.sys
 	rm -f boot/*.o
 	rm -f boot/*.efi
 	rm -f boot/*.so
-	rm -f cdrom/boot/efi.img
-	rm -f cdrom/kernel
-	rm -f cdrom/netboot cdrom/netinit
+	rm -f cdrom/fat.img cdrom/kernel cdrom/mod/* cdrom/efi/boot/bootia32.efi cdrom/ramdisk.img
+	rm -f fatbase/kernel fatbase/efi/boot/bootia32.efi
+	rm -f cdrom/netinit fatbase/netinit
 	rm -f ${KERNEL_OBJS} ${KERNEL_ASMOBJS} kernel/symbols.o kernel/symbols.S
 	rm -f base/lib/crt*.o
 	rm -f ${MODULES}

+ 1 - 1
README.md

@@ -24,7 +24,7 @@ The userspace includes a work-in-progress C standard library, the ToaruOS native
 
 ## Building
 
-First, ensure you have the necessary build tools, which are mostly the same as mainline ToaruOS: `yasm`, `xorriso`, `genext2fs` (with Debian patches), `python`.
+First, ensure you have the necessary build tools, which are mostly the same as mainline ToaruOS: `yasm`, `xorriso`, `genext2fs` (with Debian patches), `python`, and `gnu-efi` to build the EFI bootloader (I'll explore implementing necessary headers and functionality myself in the future, but for now just pull in gnu-efi and make my life easier).
 
 Run `make` and you will be prompted to build a toolchain. Reply `y` and allow the toolchain to build.
 

+ 30 - 0
util/mkdisk.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+OUT=$1
+IN=$2
+
+OUTDIR=`dirname $1`
+
+rm -f $OUT
+fallocate -l 64M $OUT
+mkfs.fat $OUT
+
+#echo "Turning $IN into $OUT"
+
+for i in $(find $IN)
+do
+    if [[ $i == $IN ]]; then
+        continue
+    fi
+    OUT_FILE=`echo $i | sed s"/^$IN/$OUTDIR/"`
+    IN_FILE=`echo $i | sed s"/^$IN//"`
+    #echo $IN_FILE  $OUT_FILE
+    if [ -d "$i" ]; then
+        mmd -i $OUT $IN_FILE
+        mkdir -p $OUT_FILE
+    else
+        mcopy -i $OUT $i '::'$IN_FILE
+        touch $OUT_FILE
+    fi
+done
+

+ 310 - 0
util/update-extents.py

@@ -0,0 +1,310 @@
+import array
+import struct
+
+def read_struct(fmt,buf,offset):
+    out, = struct.unpack_from(fmt,buf,offset)
+    return out, offset + struct.calcsize(fmt)
+
+class ISO(object):
+
+    def __init__(self, path):
+        with open(path, 'rb') as f:
+            tmp = f.read()
+            self.data = array.array('c', tmp)
+        self.sector_size = 2048
+        o = 0x10 * self.sector_size
+        self.type,             o = read_struct('B',self.data,o)
+        self.id,               o = read_struct('5s',self.data,o)
+        self.version,          o = read_struct('B',self.data,o)
+        _unused0,              o = read_struct('B',self.data,o)
+        self.system_id,        o = read_struct('32s',self.data,o)
+        self.volume_id,        o = read_struct('32s',self.data,o)
+        _unused1,              o = read_struct('8s',self.data,o)
+        self.volume_space_lsb, o = read_struct('<I',self.data,o)
+        self.volume_space_msb, o = read_struct('>I',self.data,o)
+        _unused2,              o = read_struct('32s',self.data,o)
+        self.volume_set_lsb,   o = read_struct('<H',self.data,o)
+        self.volume_set_msb,   o = read_struct('>H',self.data,o)
+        self.volume_seq_lsb,   o = read_struct('<H',self.data,o)
+        self.volume_seq_msb,   o = read_struct('>H',self.data,o)
+        self.logical_block_size_lsb,   o = read_struct('<H',self.data,o)
+        self.logical_block_size_msb,   o = read_struct('>H',self.data,o)
+        self.path_table_size_lsb,   o = read_struct('<I',self.data,o)
+        self.path_table_size_msb,   o = read_struct('>I',self.data,o)
+        self.path_table_lsb,   o = read_struct('<I',self.data,o)
+        self.optional_path_table_lsb,   o = read_struct('<I',self.data,o)
+        self.path_table_msb,   o = read_struct('>I',self.data,o)
+        self.optional_path_table_msb,   o = read_struct('>I',self.data,o)
+        _offset = o
+        self.root_dir_entry, o = read_struct('34s',self.data,o)
+
+        self.root = ISOFile(self,_offset)
+        self._cache = {}
+
+    def get_file(self, path):
+        if path == '/':
+            return self.root
+        else:
+            if path in self._cache:
+                return self._cache[path]
+            units = path.split('/')
+            units = units[1:] # remove root
+            me = self.root
+            for i in units:
+                next_file = me.find(i)
+                if not next_file:
+                    me = None
+                    break
+                else:
+                    me = next_file
+            self._cache[path] = me
+            return me
+
+class ISOFile(object):
+
+    def __init__(self, iso, offset):
+        self.iso = iso
+        self.offset = offset
+
+        o = offset
+        self.length,            o = read_struct('B', self.iso.data, o)
+        if not self.length:
+            return
+        self.ext_length,        o = read_struct('B', self.iso.data, o)
+        self.extent_start_lsb,  o = read_struct('<I',self.iso.data, o)
+        self.extent_start_msb,  o = read_struct('>I',self.iso.data, o)
+        self.extent_length_lsb, o = read_struct('<I',self.iso.data, o)
+        self.extent_length_msb, o = read_struct('>I',self.iso.data, o)
+
+        self.date_data, o = read_struct('7s', self.iso.data, o)
+
+        self.flags, o = read_struct('b', self.iso.data, o)
+        self.interleave_units, o = read_struct('b', self.iso.data, o)
+        self.interleave_gap, o = read_struct('b', self.iso.data, o)
+
+        self.volume_seq_lsb, o = read_struct('<H',self.iso.data, o)
+        self.volume_seq_msb, o = read_struct('>H',self.iso.data, o)
+
+        self.name_len, o = read_struct('b', self.iso.data, o)
+        self.name, o = read_struct('{}s'.format(self.name_len), self.iso.data, o)
+
+    def write_extents(self):
+        struct.pack_into('<I', self.iso.data, self.offset + 2, self.extent_start_lsb)
+        struct.pack_into('>I', self.iso.data, self.offset + 6, self.extent_start_lsb)
+        struct.pack_into('<I', self.iso.data, self.offset + 10, self.extent_length_lsb)
+        struct.pack_into('>I', self.iso.data, self.offset + 14, self.extent_length_lsb)
+
+    def readable_name(self):
+        if not ';' in self.name:
+            return self.name.lower()
+        else:
+            tmp, _ = self.name.split(';')
+            return tmp.lower()
+
+
+    def list(self):
+        sectors = self.iso.data[self.extent_start_lsb * self.iso.sector_size: self.extent_start_lsb * self.iso.sector_size+ 3 * self.iso.sector_size]
+        offset = 0
+
+        while 1:
+            f = ISOFile(self.iso, self.extent_start_lsb * self.iso.sector_size + offset)
+            yield f
+            offset += f.length
+            if not f.length:
+                break
+
+    def find(self, name):
+        sectors = self.iso.data[self.extent_start_lsb * self.iso.sector_size: self.extent_start_lsb * self.iso.sector_size+ 3 * self.iso.sector_size]
+        offset = 0
+        if '.' in name and len(name.split('.')[0]) > 8:
+            a, b = name.split('.')
+            name = a[:8] + '.' + b
+        if '-' in name:
+            name = name.replace('-','_')
+        while 1:
+            f = ISOFile(self.iso, self.extent_start_lsb * self.iso.sector_size + offset)
+            if not f.length:
+                if offset < self.extent_length_lsb:
+                    offset += 1
+                    continue
+                else:
+                    break
+            if ';' in f.name:
+                tmp, _ = f.name.split(';')
+                if tmp.endswith('.'):
+                    tmp = tmp[:-1]
+                if tmp.lower() == name.lower():
+                    return f
+            elif f.name.lower() == name.lower():
+                return f
+            offset += f.length
+        return None
+
+class FAT(object):
+
+    def __init__(self, iso, offset):
+        self.iso = iso
+        self.offset = offset
+
+        #print self.offset
+
+        self.bytespersector,    _ = read_struct('H', self.iso.data, offset + 11)
+        self.sectorspercluster, _ = read_struct('B', self.iso.data, offset + 13)
+        self.reservedsectors,   _ = read_struct('H', self.iso.data, offset + 14)
+        self.numberoffats,      _ = read_struct('B', self.iso.data, offset + 16)
+        self.numberofdirs,      _ = read_struct('H', self.iso.data, offset + 17)
+        self.fatsize, _ = read_struct('H', self.iso.data, offset + 22)
+
+        #print self.bytespersector, self.sectorspercluster
+
+        self.root_dir_sectors = (self.numberofdirs * 32 + (self.bytespersector - 1)) / self.bytespersector
+        #print "root_dir_sectors", self.root_dir_sectors
+        self.first_data_sector = self.reservedsectors + (self.numberoffats * self.fatsize) + self.root_dir_sectors
+        #print "resrved", self.reservedsectors
+        #print "fats, fatsize", self.numberoffats, self.fatsize
+        #print "first_data_sector_", self.first_data_sector
+        self.root_sector= self.first_data_sector - self.root_dir_sectors
+        #print self.first_data_sector - self.root_dir_sectors, self.root_sector * self.bytespersector
+        self.root = FATDirectory(self, self.offset + self.root_sector * self.bytespersector)
+
+    def get_offset(self, cluster):
+        return self.offset + ((cluster - 2) * self.sectorspercluster + self.first_data_sector) * self.bytespersector
+
+    def get_file(self, path):
+        units = path.split('/')
+        units = units[1:]
+
+        me = self.root
+        out = None
+        for i in units:
+            #print 'trav', i, me
+            for fatfile in me.list():
+                if fatfile.readable_name() == i:
+                    me = fatfile.to_dir()
+                    out = fatfile
+                    break
+            else:
+                return None
+        return out
+
+class FATDirectory(object):
+
+    def __init__(self, fat, offset):
+
+        self.fat = fat
+        self.offset = offset
+
+    def list(self):
+
+        o = self.offset
+        while 1:
+            out = FATFile(self.fat, o)
+            if out.name != '\0\0\0\0\0\0\0\0':
+                yield out
+            else:
+                break
+            o += out.size
+
+
+class FATFile(object):
+
+    def __init__(self, fat, offset):
+
+        self.fat = fat
+        self.offset = offset
+        self.magic_long = None
+        self.size = 0
+        self.long_name = ''
+
+        o = self.offset
+        self.actual_offset = o
+
+        self.attrib,     _ = read_struct('B',self.fat.iso.data,o+11)
+
+        while (self.attrib & 0x0F) == 0x0F:
+            # Long file name entry
+            tmp = read_struct('10s',self.fat.iso.data,o+1)[0]
+            tmp += read_struct('12s',self.fat.iso.data,o+14)[0]
+            tmp += read_struct('4s',self.fat.iso.data,o+28)[0]
+            tmp = "".join([x for x in tmp[::2] if x != '\xFF']).strip('\x00')
+            self.long_name = tmp + self.long_name
+            self.size += 32
+            o = self.offset + self.size
+            self.actual_offset = o
+            self.attrib,     _ = read_struct('B',self.fat.iso.data,o+11)
+
+        o = self.offset + self.size
+
+        self.name,       o = read_struct('8s',self.fat.iso.data,o)
+        self.ext,        o = read_struct('3s',self.fat.iso.data,o)
+        self.attrib,     o = read_struct('B',self.fat.iso.data,o)
+        self.userattrib, o = read_struct('B',self.fat.iso.data,o)
+        self.undelete,   o = read_struct('b',self.fat.iso.data,o)
+        self.createtime, o = read_struct('H',self.fat.iso.data,o)
+        self.createdate, o = read_struct('H',self.fat.iso.data,o)
+        self.accessdate, o = read_struct('H',self.fat.iso.data,o)
+        self.clusterhi,  o = read_struct('H',self.fat.iso.data,o)
+        self.modifiedti, o = read_struct('H',self.fat.iso.data,o)
+        self.modifiedda, o = read_struct('H',self.fat.iso.data,o)
+        self.clusterlow, o = read_struct('H',self.fat.iso.data,o)
+        self.filesize,   o = read_struct('I',self.fat.iso.data,o)
+
+        self.size += 32
+
+        self.cluster = (self.clusterhi << 16) + self.clusterlow
+
+    def is_dir(self):
+        return bool(self.attrib & 0x10)
+
+    def is_long(self):
+        return bool((self.attrib & 0x0F) == 0x0F)
+
+    def to_dir(self):
+        return FATDirectory(self.fat, self.fat.get_offset(self.cluster))
+
+    def get_offset(self):
+        return self.fat.get_offset(self.cluster)
+
+    def readable_name(self):
+        if self.long_name:
+            return self.long_name
+        if self.ext.strip():
+            return (self.name.strip() + '.' + self.ext.strip()).lower()
+        else:
+            return self.name.strip().lower()
+
+
+image = ISO('image.iso')
+fat = image.root.find('FAT.IMG')
+
+fatfs = FAT(image, fat.extent_start_lsb * image.sector_size)
+
+def process(fatfile, path):
+    if fatfile.is_long():
+        return
+    if fatfile.readable_name() == '.':
+        return
+    if fatfile.readable_name() == '..':
+        return
+    if fatfile.is_dir():
+        #print path + fatfile.readable_name(), "is a directory"
+        for i in fatfile.to_dir().list():
+            process(i, path + fatfile.readable_name() + '/')
+    else:
+        #print path + fatfile.readable_name(), "is a file"
+        cdfile = image.get_file(path + fatfile.readable_name())
+        if not cdfile:
+            print "Warning:", fatfile.readable_name(), "not found in ISO"
+        else:
+            #print fatfile.get_offset() / 2048, fatfile.filesize
+            cdfile.extent_start_lsb = fatfile.get_offset() / 2048
+            cdfile.extent_length_lsb = fatfile.filesize
+            cdfile.write_extents()
+
+
+for i in fatfs.root.list():
+    process(i,'/')
+
+with open('image.iso','wb') as f:
+    f.write(image.data)
+