Discussion:
[PATCH] Generate all build files with single program.
(too old to reply)
Georgi Chorbadzhiyski
2012-03-16 22:05:06 UTC
Permalink
This patch removes all external tools from the build system. Now
everything in generated/ directory is build with single program.
The effect is that the build no longer requires sed, tr, sort and
python (for rebuilding the help texts). There is no need to ship
generated/help.h since it can be rebuild on the host without python.

There is also a nice speedup. On my machine building allyesconfig
is ~1.8 seconds faster, it goes down from 11.3s to 0m9.5s.
---
Makefile | 7 +-
scripts/config2help.py | 54 ---------
scripts/config2help.sh | 54 ---------
scripts/genconf.c | 309 ++++++++++++++++++++++++++++++++++++++++++++++++
scripts/genconfig.sh | 10 ++-
scripts/make.sh | 88 +--------------
6 files changed, 324 insertions(+), 198 deletions(-)
delete mode 100755 scripts/config2help.py
delete mode 100755 scripts/config2help.sh
create mode 100644 scripts/genconf.c

diff --git a/Makefile b/Makefile
index 7079f82..4aec0ff 100644
--- a/Makefile
+++ b/Makefile
@@ -17,6 +17,9 @@ generated/Config.in: toys/*.c scripts/genconfig.sh

HOSTCC?=cc

+.config:
+ $(MAKE) defconfig
+
# Development targets
baseline: toybox_unstripped
@cp toybox_unstripped toybox_old
@@ -42,10 +45,10 @@ uninstall:
clean::
rm -rf toybox toybox_unstripped generated/config.h generated/Config.in \
generated/newtoys.h generated/globals.h instlist testdir \
- generated/Config.probed
+ generated/Config.probed generated/help.h generated/build_files

distclean: clean
- rm -f toybox_old .config* generated/help.h
+ rm -f toybox_old .config* generated/genconf

test: tests

diff --git a/scripts/config2help.py b/scripts/config2help.py
deleted file mode 100755
index 2573d08..0000000
--- a/scripts/config2help.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/python
-
-import os,sys
-
-def zapquotes(str):
- if str[0]=='"': str = str[1:str.rfind('"')]
- return str
-
-def escapequotes(str):
- return str.strip().replace("\\","\\\\").replace('"','\\"')
-
-helplen = morelines = 0
-out = sys.stdout
-
-def readfile(filename):
- global helplen, morelines
- #sys.stderr.write("Reading %s\n" % filename)
- try:
- lines = open(filename).read().split("\n")
- except IOError:
- sys.stderr.write("File %s missing\n" % filename)
- return
- config = None
- description = None
- for i in lines:
- if helplen:
- i = i.expandtabs()
- if not len(i) or i[:helplen].isspace():
- if morelines: out.write('\\n')
- morelines = 1
- out.write(escapequotes(i))
- continue
- else:
- helplen = morelines = 0
- out.write('"\n')
-
- words = i.strip().split(None,1)
- if not len(words): continue
-
- if words[0] in ("config", "menuconfig"):
- config = words[1]
- description = ""
- elif words[0] in ("bool", "boolean", "tristate", "string", "hex", "int"):
- if len(words)>1: description = zapquotes(words[1])
- elif words[0]=="prompt":
- description = htmlescape(zapquotes(words[1]))
- elif words[0] in ("help", "---help---"):
- out.write('#define help_%s "' % config.lower())
- helplen = len(i[:i.find(words[0])].expandtabs())
- elif words[0] == "source": readfile(zapquotes(words[1]))
- elif words[0] in ("default","depends", "select", "if", "endif", "#", "comment", "menu", "endmenu"): pass
-
-readfile(sys.argv[1])
-if helplen: out.write('"\n')
diff --git a/scripts/config2help.sh b/scripts/config2help.sh
deleted file mode 100755
index 63d6c6f..0000000
--- a/scripts/config2help.sh
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/bin/bash -e
-
-function firstmajor() {
- declare -i n j=$1
- test $j -gt 0 || return -1
- for j in $@; do
- if [ $j -gt $1 ]; then
- echo $j
- return 0
- fi
- done
- return 1
-}; export -f firstmajor
-
-function print_h() {
- declare -i i c=$2 s=$3 e=$4
- local str="$(echo "$1" | head -n$c | tail -n1 | sed -e "s,config[\t ]*,,")"
- echo -n "#define help_"$str" \"" | tr [A-Z] [a-z]
- echo -n "$1\\n" | head -n$e | tail -n$[e-s+1] | sed -e "s,\$,\r," | tr \\n\\r n\\
- echo \"
-}; export -f print_h
-
-file="$1"
-if test "$0" != "bash"; then
- if test -r "$file"; then
-# echo "$file..." >&2
- filetxt="$(sed -e "s,^[\t ]*,," -e "s,\([\"\\\\]\),\\\\\\1,g" "$file")"
- helps=$(echo "$filetxt" | egrep -ne "^help *" -e "^---help--- *" | cut -d\: -f1)
- configs=$(echo "$filetxt" | egrep -ne "^config *" | cut -d\: -f1)
- endmenus=$(echo "$filetxt" | egrep -ne "^endmenu *" | cut -d\: -f1)
- let last=$(echo "$filetxt" | wc -l)+2
-
- declare -i i c s e
- for i in $configs; do
-# echo -n "c:$i" >&2
- s=$(firstmajor $i $helps)
- test $s -gt 0
- e=$(firstmajor $s $configs || firstmajor $s $endmenus $last)
- let s++ e-=2
- test $e -ge $s
-# echo " s:$s e:$e" >&2
- print_h "$filetxt" $i $s $e
- done
- for fle in $(cat "$file" | egrep -e "^[ \t]*source " | sed -e "s,^[ \t]*source *\(.*\),\\1,"); do
- $0 $fle
- done
- else
- echo
- echo "USAGE EXAMPLE: $(basename $0) Config.in > generated/help.h"
- echo
- false
- fi | sed -e "s,\\\\n\\\\n\"$,\\\\n\","
-fi
-
diff --git a/scripts/genconf.c b/scripts/genconf.c
new file mode 100644
index 0000000..8346dee
--- /dev/null
+++ b/scripts/genconf.c
@@ -0,0 +1,309 @@
+/* vi: set ts=4 :*/
+/*
+ * Generate toybox generated/{Config.in,config.h,globals.h,newtoys.h,help.h}
+ *
+ * Copyright 2012 Georgi Chorbadzhiyski <georgi at unixsol.org>
+ */
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <unistd.h>
+#include <errno.h>
+
+struct strbuf {
+ unsigned int len;
+ unsigned int nchars;
+ char *data;
+};
+
+struct strbuf newtoys_h;
+struct strbuf config_h;
+struct strbuf config_in;
+struct strbuf help_h;
+struct strbuf globals_h;
+struct strbuf globals_union;
+struct strbuf build_files;
+
+int is_toy;
+int in_config;
+int in_help;
+int in_globals;
+
+char *line = NULL;
+size_t linelen;
+char *pos, *toyname, *conf;
+int i;
+char tmp[1024];
+
+
+FILE *f;
+
+FILE *xfopen(char *filename, char *mode) {
+ FILE *fh = fopen(filename, mode);
+ if (!fh) {
+ fprintf(stderr, "fopen(%s, %s): %s\n", filename, mode, strerror(errno));
+ exit(-1);
+ }
+ return fh;
+}
+
+void strbuf_add(struct strbuf *s, char *text) {
+ int len = strlen(text);
+ if (!len)
+ return;
+ if (!s->data) {
+ s->nchars = 1024;
+ s->data = calloc(1, s->nchars);
+ }
+ s->len += len;
+ if (s->len >= s->nchars) {
+ s->nchars += s->len + 1;
+ s->data = realloc(s->data, s->nchars);
+ if (!s->data) {
+ fprintf(stderr, "realloc(%d): %s\n", s->nchars, strerror(errno));
+ exit(-1);
+ }
+ }
+ strcat(s->data, text);
+}
+
+__attribute__ ((format(printf, 2, 3)))
+void strbuf_printf(struct strbuf *s, const char *fmt, ...) {
+ char line[4096];
+
+ va_list args;
+ va_start(args, fmt);
+ vsnprintf(line, sizeof(line) - 1, fmt, args);
+ va_end(args);
+
+ line[sizeof(line) - 1] = '\0';
+ strbuf_add(s, line);
+}
+
+void strbuf_add_escaped(struct strbuf *s, char *text) {
+ int len = strlen(text), i;
+ if (len == 0)
+ strbuf_add(s, "\\n");
+ for (i = 0; i < len; i++) {
+ char c[2] = { text[i], 0 };
+ switch (text[i]) {
+ case '\n': strbuf_add(s, "\\n"); break;
+ case '\t': strbuf_add(s, " "); break;
+ case '\\': strbuf_add(s, "\\\\"); break;
+ case '"' : strbuf_add(s, "\\\""); break;
+ default : strbuf_add(s, c); break;
+ }
+ }
+}
+
+void strbuf_dump(struct strbuf *s, char *filename, char *mode) {
+ FILE *fh;
+ if (!s->data) {
+ unlink(filename);
+ return;
+ }
+ fh = xfopen(filename, mode);
+ fwrite(s->data, s->len, 1, fh);
+ fclose(fh);
+ free(s->data);
+}
+
+void parse_help(char *filename) {
+ int i, nitems = 0;
+ char *config_name = NULL;
+
+ f = xfopen(filename, "r");
+ while (getline(&line, &linelen, f) > 0) {
+ if (line[0] == '#')
+ continue;
+ if (strncmp(line, "config ", 7) == 0) {
+ in_config = 1;
+ in_help = 0;
+ free(config_name);
+ config_name = strdup(line + 7);
+ int config_len = strlen(config_name);
+ for (i = 0; i < config_len; i++) {
+ config_name[i] = tolower((unsigned char)config_name[i]);
+ if (config_name[i] == '\n')
+ config_name[i] = '\0';
+ }
+ // Close already started help line
+ if (nitems)
+ strbuf_add(&help_h, "\"\n");
+ continue;
+ }
+ if (strncmp(line, "endmenu", 7) == 0) {
+ in_config = 0;
+ in_help = 0;
+ }
+ if (!in_config)
+ continue;
+
+ if (!in_help) {
+ if (strncmp(line, "\thelp", 5) == 0 || strncmp(line, " help", 8) == 0) {
+ in_help = 1;
+ nitems++;
+ strbuf_printf(&help_h, "#define help_%s \"", config_name);
+ }
+ continue;
+ } else {
+ char *tmpline = line;
+ while (isspace(*tmpline))
+ tmpline++;
+ strbuf_add_escaped(&help_h, tmpline);
+ continue;
+ }
+ }
+ strbuf_add(&help_h, "\"\n");
+
+ fclose(f);
+
+ free(config_name);
+}
+
+void generate_headers(int argc, char **argv) {
+ // Parse toys
+ strbuf_add(&newtoys_h, "NEWTOY(toybox, NULL, 0)\n");
+ strbuf_add(&globals_union, "extern union global_union {\n");
+
+ for (i = 1; i < argc; i++) {
+ f = xfopen(argv[i], "r");
+
+ toyname = argv[i];
+ pos = strrchr(toyname, '/');
+ if (pos)
+ toyname = pos + 1; // Remove path
+ pos = strstr(toyname, ".c");
+ if (pos) // Remove extension
+ pos[0] = '\0';
+
+ strbuf_printf(&globals_h, "// %s.c\n\n", argv[i]);
+
+ while (getline(&line, &linelen, f) > 0) {
+ // Parse toys
+ if (strncmp(line, "USE_", 4) == 0) {
+ is_toy = 1;
+ strbuf_add(&newtoys_h, line);
+ continue;
+ }
+ if (!is_toy)
+ continue;
+
+ // Parse config
+ if (!in_config) {
+ if (strncmp(line, "config ", 7) == 0) {
+ in_config = 1;
+ strbuf_printf(&config_in, "# %s.c\n", argv[i]);
+ strbuf_add(&config_in, line);
+ continue;
+ }
+ } else {
+ if (strncmp(line, "*/", 2) == 0) {
+ strbuf_add(&config_in, "\n");
+ in_config = 0;
+ continue;
+ } else {
+ strbuf_add(&config_in, line);
+ }
+ }
+
+ // Parse globals
+ if (!in_globals) {
+ if (strncmp(line, "DEFINE_GLOBALS(", 15) == 0) {
+ strbuf_printf(&globals_h, "struct %s_data {\n", toyname);
+ strbuf_printf(&globals_union, "\tstruct %s_data %s;\n", toyname, toyname);
+ in_globals = 1;
+ continue;
+ }
+ } else {
+ if (strncmp(line, ")", 1) == 0) {
+ strbuf_add(&globals_h, "};\n");
+ in_globals = 0;
+ break;
+ } else {
+ strbuf_add(&globals_h, line);
+ }
+ }
+ }
+ fclose(f);
+ }
+
+ strbuf_add(&globals_union, "} this;\n");
+
+ strbuf_dump(&config_in, "generated/Config.in", "w");
+ strbuf_dump(&globals_h, "generated/globals.h", "w");
+ strbuf_dump(&globals_union, "generated/globals.h", "a");
+ strbuf_dump(&newtoys_h, "generated/newtoys.h", "w");
+}
+
+void generate_config_in() {
+ parse_help("Config.in");
+ parse_help("generated/Config.in");
+ strbuf_dump(&help_h, "generated/help.h", "w");
+}
+
+void parse_config() {
+ // Configu do not exists, create dummy .config
+ if (!access(".config", R_OK) == 0) {
+ f = xfopen(".config", "w");
+ fclose(f);
+ return;
+ }
+
+ // Parse .config
+ f = xfopen(".config", "r");
+ while (getline(&line, &linelen, f) > 0) {
+ if (strncmp(line, "CONFIG_", 7) == 0) {
+ // Config is ON
+ conf = line + 7;
+ pos = strchr(conf, '=');
+ if (pos) pos[0] = '\0';
+ strbuf_printf(&config_h, "#define CFG_%s 1\n", conf);
+ strbuf_printf(&config_h, "#define USE_%s(...) __VA_ARGS__\n", conf);
+ if (strncmp(conf, "TOYBOX_", 7) != 0) {
+ // Generate build_files
+ int config_len = strlen(conf);
+ for (i = 0; i < config_len; i++) {
+ conf[i] = tolower((unsigned char)conf[i]);
+ if (conf[i] == '\n' || conf[i] == ' ' || conf[i] == '=')
+ conf[i] = '\0';
+ }
+ snprintf(tmp, sizeof(tmp) - 1, "toys/%s.c", conf);
+ tmp[sizeof(tmp) - 1] = '\0';
+ if (access(tmp, R_OK) == 0)
+ strbuf_printf(&build_files, "%s\n", tmp);
+ }
+ continue;
+ }
+ if (strncmp(line, "# CONFIG_", 9) == 0) {
+ // Config is OFF
+ conf = line + 9;
+ pos = strchr(conf, ' ');
+ if (pos) pos[0] = '\0';
+ strbuf_printf(&config_h, "#define CFG_%s 0\n", conf);
+ strbuf_printf(&config_h, "#define USE_%s(...)\n", conf);
+ continue;
+ }
+ }
+ fclose(f);
+ strbuf_dump(&config_h, "generated/config.h", "w");
+ strbuf_dump(&build_files, "generated/build_files", "w");
+}
+
+
+int main(int argc, char **argv) {
+ if (argc < 2) {
+ fprintf(stderr, "Usage: %s toys/*.c\n", argv[0]);
+ exit(1);
+ }
+
+ generate_headers(argc, argv);
+ generate_config_in();
+ parse_config();
+
+ free(line);
+
+ return 0;
+}
diff --git a/scripts/genconfig.sh b/scripts/genconfig.sh
index f677954..c562e37 100755
--- a/scripts/genconfig.sh
+++ b/scripts/genconfig.sh
@@ -34,4 +34,12 @@ genconfig()
}

probeconfig > generated/Config.probed || rm generated/Config.probed
-genconfig > generated/Config.in || rm generated/Config.in
+
+if [ ! -x generated/genconf ]
+then
+ echo "Building generated/genconf"
+ $HOSTCC -ggdb -Wall -Wextra -Wno-sign-compare -I . -o generated/genconf scripts/genconf.c || exit 1
+fi
+
+echo "Extract configuration information from toys/*.c files..."
+generated/genconf toys/*.c || exit 1
diff --git a/scripts/make.sh b/scripts/make.sh
index f5bf0fd..e9afa68 100755
--- a/scripts/make.sh
+++ b/scripts/make.sh
@@ -4,95 +4,9 @@

source ./configure

-echo "Extract configuration information from toys/*.c files..."
scripts/genconfig.sh

-echo "Generate headers from toys/*.c..."
-
-# Create a list of all the applets toybox can provide. Note that the first
-# entry is out of order on purpose (the toybox multiplexer applet must be the
-# first element of the array). The rest must be sorted in alphabetical order
-# for fast binary search.
-
-function newtoys()
-{
- for i in toys/*.c
- do
- sed -n -e '1,/^config [A-Z]/s/^USE_/&/p' $i || exit 1
- done
-}
-echo "NEWTOY(toybox, NULL, 0)" > generated/newtoys.h
-newtoys | sed 's/\(.*TOY(\)\([^,]*\),\(.*\)/\2 \1\2,\3/' | sort -k 1,1 \
- | sed 's/[^ ]* //' >> generated/newtoys.h
-
-# Extract global structure definitions from toys/*.c
-
-function getglobals()
-{
- for i in toys/*.c
- do
- NAME="$(echo $i | sed 's at toys/\(.*\)\.c@\1@')"
-
- echo -e "// $i\n"
- sed -n -e '/^DEFINE_GLOBALS(/,/^)/b got;b;:got' \
- -e 's/^DEFINE_GLOBALS(/struct '"$NAME"'_data {/' \
- -e 's/^)/};/' -e 'p' $i
- done
-}
-
-GLOBSTRUCT="$(getglobals)"
-(
- echo "$GLOBSTRUCT"
- echo
- echo "extern union global_union {"
- echo "$GLOBSTRUCT" | sed -n 's/struct \(.*\)_data {/ struct \1_data \1;/p'
- echo "} this;"
-) > generated/globals.h
-
-# Only recreate generated/help.h if python is installed
-if [ ! -z "$(which python)" ] && [ ! -z "$(grep 'CONFIG_HELP=y' .config)" ]
-then
- echo "Extract help text from Config.in."
- scripts/config2help.py Config.in > generated/help.h || exit 1
-fi
-
-echo "Make generated/config.h from .config."
-
-# This long and roundabout sed invocation is to make old versions of sed happy.
-# New ones have '\n' so can replace one line with two without all the branches
-# and tedious mucking about with hold space.
-
-sed -n \
- -e 's/^# CONFIG_\(.*\) is not set.*/\1/' \
- -e 't notset' \
- -e 's/^CONFIG_\(.*\)=y.*/\1/' \
- -e 't isset' \
- -e 's/^CONFIG_\([^=]*\)=\(.*\)/#define CFG_\1 \2/p' \
- -e 'd' \
- -e ':notset' \
- -e 'h' \
- -e 's/.*/#define CFG_& 0/p' \
- -e 'g' \
- -e 's/.*/#define USE_&(...)/p' \
- -e 'd' \
- -e ':isset' \
- -e 'h' \
- -e 's/.*/#define CFG_& 1/p' \
- -e 'g' \
- -e 's/.*/#define USE_&(...) __VA_ARGS__/p' \
- .config > generated/config.h || exit 1
-
-# Extract a list of toys/*.c files to compile from the data in ".config" with
-# sed, sort, and tr:
-
-# 1) Grab the XXX part of all CONFIG_XXX entries, removing everything after the
-# second underline
-# 2) Sort the list, keeping only one of each entry.
-# 3) Convert to lower case.
-# 4) Remove toybox itself from the list (as that indicates global symbols).
-# 5) Add "toys/" prefix and ".c" suffix.
-
-TOYFILES=$(cat .config | sed -nre 's/^CONFIG_(.*)=y/\1/;t skip;b;:skip;s/_.*//;p' | sort -u | tr A-Z a-z | grep -v '^toybox$' | sed 's@\(.*\)@toys/\1.c@' )
+TOYFILES=$(cat generated/build_files 2>/dev/null)

echo "Compile toybox..."
--
1.7.5.1
Rob Landley
2012-03-17 02:57:45 UTC
Permalink
Post by Georgi Chorbadzhiyski
This patch removes all external tools from the build system. Now
everything in generated/ directory is build with single program.
The effect is that the build no longer requires sed, tr, sort and
python (for rebuilding the help texts). There is no need to ship
generated/help.h since it can be rebuild on the host without python.
Cool!

Generating help.h in C (and eliminating the need for python) is
something I've been meaning to do for a while. But building the list of
C files to compile... in C... seems unnecessarily brittle somehow.
Post by Georgi Chorbadzhiyski
There is also a nice speedup. On my machine building allyesconfig
is ~1.8 seconds faster, it goes down from 11.3s to 0m9.5s.
---
Makefile | 7 +-
scripts/config2help.py | 54 ---------
scripts/config2help.sh | 54 ---------
scripts/genconf.c | 309 ++++++++++++++++++++++++++++++++++++++++++++++++
scripts/genconfig.sh | 10 ++-
scripts/make.sh | 88 +--------------
6 files changed, 324 insertions(+), 198 deletions(-)
delete mode 100755 scripts/config2help.py
delete mode 100755 scripts/config2help.sh
create mode 100644 scripts/genconf.c
diff --git a/Makefile b/Makefile
index 7079f82..4aec0ff 100644
--- a/Makefile
+++ b/Makefile
@@ -17,6 +17,9 @@ generated/Config.in: toys/*.c scripts/genconfig.sh
HOSTCC?=cc
+ $(MAKE) defconfig
+
# Development targets
baseline: toybox_unstripped
@cp toybox_unstripped toybox_old
rm -rf toybox toybox_unstripped generated/config.h generated/Config.in \
generated/newtoys.h generated/globals.h instlist testdir \
- generated/Config.probed
+ generated/Config.probed generated/help.h generated/build_files
distclean: clean
- rm -f toybox_old .config* generated/help.h
+ rm -f toybox_old .config* generated/genconf
test: tests
diff --git a/scripts/config2help.py b/scripts/config2help.py
deleted file mode 100755
index 2573d08..0000000
--- a/scripts/config2help.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/python
-
-import os,sys
-
- if str[0]=='"': str = str[1:str.rfind('"')]
- return str
-
- return str.strip().replace("\\","\\\\").replace('"','\\"')
-
-helplen = morelines = 0
-out = sys.stdout
-
- global helplen, morelines
- #sys.stderr.write("Reading %s\n" % filename)
- lines = open(filename).read().split("\n")
- sys.stderr.write("File %s missing\n" % filename)
- return
- config = None
- description = None
- i = i.expandtabs()
- if morelines: out.write('\\n')
- morelines = 1
- out.write(escapequotes(i))
- continue
- helplen = morelines = 0
- out.write('"\n')
-
- words = i.strip().split(None,1)
- if not len(words): continue
-
- config = words[1]
- description = ""
- if len(words)>1: description = zapquotes(words[1])
- description = htmlescape(zapquotes(words[1]))
- out.write('#define help_%s "' % config.lower())
- helplen = len(i[:i.find(words[0])].expandtabs())
- elif words[0] == "source": readfile(zapquotes(words[1]))
- elif words[0] in ("default","depends", "select", "if", "endif", "#", "comment", "menu", "endmenu"): pass
-
-readfile(sys.argv[1])
-if helplen: out.write('"\n')
diff --git a/scripts/config2help.sh b/scripts/config2help.sh
deleted file mode 100755
index 63d6c6f..0000000
--- a/scripts/config2help.sh
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/bin/bash -e
-
-function firstmajor() {
- declare -i n j=$1
- test $j -gt 0 || return -1
- if [ $j -gt $1 ]; then
- echo $j
- return 0
- fi
- done
- return 1
-}; export -f firstmajor
-
-function print_h() {
- declare -i i c=$2 s=$3 e=$4
- local str="$(echo "$1" | head -n$c | tail -n1 | sed -e "s,config[\t ]*,,")"
- echo -n "#define help_"$str" \"" | tr [A-Z] [a-z]
- echo -n "$1\\n" | head -n$e | tail -n$[e-s+1] | sed -e "s,\$,\r," | tr \\n\\r n\\
- echo \"
-}; export -f print_h
-
-file="$1"
-if test "$0" != "bash"; then
- if test -r "$file"; then
-# echo "$file..." >&2
- filetxt="$(sed -e "s,^[\t ]*,," -e "s,\([\"\\\\]\),\\\\\\1,g" "$file")"
- helps=$(echo "$filetxt" | egrep -ne "^help *" -e "^---help--- *" | cut -d\: -f1)
- configs=$(echo "$filetxt" | egrep -ne "^config *" | cut -d\: -f1)
- endmenus=$(echo "$filetxt" | egrep -ne "^endmenu *" | cut -d\: -f1)
- let last=$(echo "$filetxt" | wc -l)+2
-
- declare -i i c s e
- for i in $configs; do
-# echo -n "c:$i" >&2
- s=$(firstmajor $i $helps)
- test $s -gt 0
- e=$(firstmajor $s $configs || firstmajor $s $endmenus $last)
- let s++ e-=2
- test $e -ge $s
-# echo " s:$s e:$e" >&2
- print_h "$filetxt" $i $s $e
- done
- for fle in $(cat "$file" | egrep -e "^[ \t]*source " | sed -e "s,^[ \t]*source *\(.*\),\\1,"); do
- $0 $fle
- done
- else
- echo
- echo "USAGE EXAMPLE: $(basename $0) Config.in > generated/help.h"
- echo
- false
- fi | sed -e "s,\\\\n\\\\n\"$,\\\\n\","
-fi
-
diff --git a/scripts/genconf.c b/scripts/genconf.c
new file mode 100644
index 0000000..8346dee
--- /dev/null
+++ b/scripts/genconf.c
@@ -0,0 +1,309 @@
+/* vi: set ts=4 :*/
+/*
+ * Generate toybox generated/{Config.in,config.h,globals.h,newtoys.h,help.h}
+ *
+ * Copyright 2012 Georgi Chorbadzhiyski <georgi at unixsol.org>
+ */
+__attribute__ ((format(printf, 2, 3)))
+void strbuf_printf(struct strbuf *s, const char *fmt, ...) {
This is a purely local function, the __attribute__ tag is overkill.
Post by Georgi Chorbadzhiyski
+ char line[4096];
+
+ va_list args;
+ va_start(args, fmt);
+ vsnprintf(line, sizeof(line) - 1, fmt, args);
+ va_end(args);
+
+ line[sizeof(line) - 1] = '\0';
+ strbuf_add(s, line);
+}
+
+void strbuf_add_escaped(struct strbuf *s, char *text) {
+ int len = strlen(text), i;
+ if (len == 0)
+ strbuf_add(s, "\\n");
+ for (i = 0; i < len; i++) {
+ char c[2] = { text[i], 0 };
+ switch (text[i]) {
+ case '\n': strbuf_add(s, "\\n"); break;
+ case '\t': strbuf_add(s, " "); break;
+ case '\\': strbuf_add(s, "\\\\"); break;
+ case '"' : strbuf_add(s, "\\\""); break;
+ default : strbuf_add(s, c); break;
+ }
+ }
+}
+
+void strbuf_dump(struct strbuf *s, char *filename, char *mode) {
+ FILE *fh;
+ if (!s->data) {
+ unlink(filename);
+ return;
+ }
+ fh = xfopen(filename, mode);
+ fwrite(s->data, s->len, 1, fh);
+ fclose(fh);
+ free(s->data);
+}
+
+void parse_help(char *filename) {
+ int i, nitems = 0;
+ char *config_name = NULL;
+
+ f = xfopen(filename, "r");
+ while (getline(&line, &linelen, f) > 0) {
+ if (line[0] == '#')
+ continue;
+ if (strncmp(line, "config ", 7) == 0) {
+ in_config = 1;
+ in_help = 0;
+ free(config_name);
+ config_name = strdup(line + 7);
+ int config_len = strlen(config_name);
+ for (i = 0; i < config_len; i++) {
+ config_name[i] = tolower((unsigned char)config_name[i]);
+ if (config_name[i] == '\n')
+ config_name[i] = '\0';
+ }
+ // Close already started help line
+ if (nitems)
+ strbuf_add(&help_h, "\"\n");
+ continue;
+ }
+ if (strncmp(line, "endmenu", 7) == 0) {
+ in_config = 0;
+ in_help = 0;
+ }
+ if (!in_config)
+ continue;
+
+ if (!in_help) {
+ if (strncmp(line, "\thelp", 5) == 0 || strncmp(line, " help", 8) == 0) {
+ in_help = 1;
+ nitems++;
+ strbuf_printf(&help_h, "#define help_%s \"", config_name);
+ }
+ continue;
+ } else {
+ char *tmpline = line;
+ while (isspace(*tmpline))
+ tmpline++;
+ strbuf_add_escaped(&help_h, tmpline);
+ continue;
+ }
+ }
+ strbuf_add(&help_h, "\"\n");
I was trying to detect sub-options, and also merge the "usage:" lines.
(This turn out to be a bit fiddly, you can use the "depends" lines which
can have "depends on ONE && TWO" so there's some parsing to do, or you
can depend on CONFIG_WALRUS and CONFIG_WALRUS_SUBOPT starting with the
same prefix, which gets fiddly when CONFIG_WALRUS has multiple options
and how do you tell that CONFIG_PIVOT_ROOT isn't a subopt for CONFIG_PIVOT?)
Post by Georgi Chorbadzhiyski
+void generate_headers(int argc, char **argv) {
+ // Parse toys
+ strbuf_add(&newtoys_h, "NEWTOY(toybox, NULL, 0)\n");
+ strbuf_add(&globals_union, "extern union global_union {\n");
It matters that these be in alphabetical order (for the binary search),
and we can't depend on the filesystem doing it for us. Where are you
sorting this?

I'll have to look at this more closely in the morning...

Rob
Georgi Chorbadzhiyski
2012-03-18 16:14:19 UTC
Permalink
Post by Rob Landley
Post by Georgi Chorbadzhiyski
This patch removes all external tools from the build system. Now
everything in generated/ directory is build with single program.
The effect is that the build no longer requires sed, tr, sort and
python (for rebuilding the help texts). There is no need to ship
generated/help.h since it can be rebuild on the host without python.
Cool!
Generating help.h in C (and eliminating the need for python) is
something I've been meaning to do for a while. But building the list of
C files to compile... in C... seems unnecessarily brittle somehow.
No more than the sed usage (which I was unable to simplify and this is
was drove me to make the C version).
Post by Rob Landley
Post by Georgi Chorbadzhiyski
+__attribute__ ((format(printf, 2, 3)))
+void strbuf_printf(struct strbuf *s, const char *fmt, ...) {
This is a purely local function, the __attribute__ tag is overkill.
It is just a habit I've picked when adding new printf style varargs
functions.
Post by Rob Landley
Post by Georgi Chorbadzhiyski
+void generate_headers(int argc, char **argv) {
+ // Parse toys
+ strbuf_add(&newtoys_h, "NEWTOY(toybox, NULL, 0)\n");
+ strbuf_add(&globals_union, "extern union global_union {\n");
It matters that these be in alphabetical order (for the binary search),
and we can't depend on the filesystem doing it for us. Where are you
sorting this?
I didn't sort them. I depend on the shell doing the right thing. The program
is called like this generated/genconf toys/*.c so it should be OK. Of course
the parameters can be sorted but it seemed unnecessary since the shell is
doing that for us.
--
Georgi Chorbadzhiyski
http://georgi.unixsol.org/
Rob Landley
2012-03-18 16:55:06 UTC
Permalink
Post by Georgi Chorbadzhiyski
Post by Rob Landley
Post by Georgi Chorbadzhiyski
This patch removes all external tools from the build system. Now
everything in generated/ directory is build with single program.
The effect is that the build no longer requires sed, tr, sort and
python (for rebuilding the help texts). There is no need to ship
generated/help.h since it can be rebuild on the host without python.
Cool!
Generating help.h in C (and eliminating the need for python) is
something I've been meaning to do for a while. But building the list of
C files to compile... in C... seems unnecessarily brittle somehow.
No more than the sed usage (which I was unable to simplify and this is
was drove me to make the C version).
I'm a bit biased because I wrote the busybox sed implementation. (And
string handling in C is pretty inherently brittle...)
Post by Georgi Chorbadzhiyski
Post by Rob Landley
Post by Georgi Chorbadzhiyski
+void generate_headers(int argc, char **argv) {
+ // Parse toys
+ strbuf_add(&newtoys_h, "NEWTOY(toybox, NULL, 0)\n");
+ strbuf_add(&globals_union, "extern union global_union {\n");
It matters that these be in alphabetical order (for the binary search),
and we can't depend on the filesystem doing it for us. Where are you
sorting this?
I didn't sort them. I depend on the shell doing the right thing. The program
is called like this generated/genconf toys/*.c so it should be OK. Of course
the parameters can be sorted but it seemed unnecessary since the shell is
doing that for us.
Actually the shell isn't. The filesystem is. (I've hit this one
before, it's not reliable.)

Also, sorting the .c files isn't sufficient, you have to sort the
_output_ array, because one .c file can produce more than one array
entry (sh and toysh, netcat and nc...)

Rob
--
GNU/Linux isn't: Linux=GPLv2, GNU=GPLv3+, they can't share code.
Either it's "mere aggregation", or a license violation. Pick one.
Georgi Chorbadzhiyski
2012-03-18 21:17:13 UTC
Permalink
Post by Rob Landley
Post by Georgi Chorbadzhiyski
Post by Rob Landley
Post by Georgi Chorbadzhiyski
+void generate_headers(int argc, char **argv) {
+ // Parse toys
+ strbuf_add(&newtoys_h, "NEWTOY(toybox, NULL, 0)\n");
+ strbuf_add(&globals_union, "extern union global_union {\n");
It matters that these be in alphabetical order (for the binary search),
and we can't depend on the filesystem doing it for us. Where are you
sorting this?
I didn't sort them. I depend on the shell doing the right thing. The program
is called like this generated/genconf toys/*.c so it should be OK. Of course
the parameters can be sorted but it seemed unnecessary since the shell is
doing that for us.
Actually the shell isn't. The filesystem is. (I've hit this one
before, it's not reliable.)
Also, sorting the .c files isn't sufficient, you have to sort the
_output_ array, because one .c file can produce more than one array
entry (sh and toysh, netcat and nc...)
I'll send an updated patch tomorrow.
--
Georgi Chorbadzhiyski
http://georgi.unixsol.org/
Rob Landley
2012-03-19 13:35:16 UTC
Permalink
Post by Georgi Chorbadzhiyski
Post by Rob Landley
Post by Georgi Chorbadzhiyski
Post by Rob Landley
Post by Georgi Chorbadzhiyski
+void generate_headers(int argc, char **argv) {
+ // Parse toys
+ strbuf_add(&newtoys_h, "NEWTOY(toybox, NULL, 0)\n");
+ strbuf_add(&globals_union, "extern union global_union {\n");
It matters that these be in alphabetical order (for the binary search),
and we can't depend on the filesystem doing it for us. Where are you
sorting this?
I didn't sort them. I depend on the shell doing the right thing. The program
is called like this generated/genconf toys/*.c so it should be OK. Of course
the parameters can be sorted but it seemed unnecessary since the shell is
doing that for us.
Actually the shell isn't. The filesystem is. (I've hit this one
before, it's not reliable.)
Also, sorting the .c files isn't sufficient, you have to sort the
_output_ array, because one .c file can produce more than one array
entry (sh and toysh, netcat and nc...)
I'll send an updated patch tomorrow.
The C command I was working on had several goals:

1) Remember when Daniel Walter did groups and whoami as reskins of id?
The current infrastructure supports aliases (two commands with the same
behavior), but does _not_ support two commands in the same file with
different main() functions and help text.

2) I'm trying to add the nbd-client.c I wrote back in 2010, and with
config NBD_CLIENT the build is trying to include "nbd.c".

3) I want to combine the help text and usage: lines, as explained here
earlier.

4) I'd like to be able to build individual commands without necessarily
going through the config infrastructure. (That way I could build the
toybox sed/sort/shell commands to run the rest of the build even if the
host tools are strange, but it's also nice to be able to build a
non-swiss-army-knife version fo the binaries. I did "make individual"
for busybox long ago for similar reasons.)

5) I was hoping the config-parsing infrastructure I made could also act
as the start of a kconfig replacement. If the help text merging bits
have to understand the kconfig language thoroughly enough to parse "XXX
&& YYY" style dependencies (and without dependency tracking the rules on
what's an option to a command and what's a new command get REALLY
brittle in the presence of a single file with multiple commands that can
be named "pivot_root" and "newfs_msdos" and so on.)

Merging this patch means I abandoning the (admittedly unfinished) work
I've done towards those goals, which would be fine if it addressed any
of them, but it doesn't.

It replaces it with a build infrastructure I've read through twice and
still not quite wrapped my head around yet (your buffer infrastructure
is doing what again: seems kind a overkill...). It brings back bad
associations to the old busybox "makedep" command. Its primary goal
seems to be making the project compile faster, which isn't exactly a
priority for me. (The current build doesn't even support smp
compilation, that's lower-hanging fruit than rewriting the shell scripts
in C.)

If you'd done a patch that was _just_ changing the help text and leaving
the rest of it alone, I'd already have merged it and worried about
fixing up the rest I need later. I am _very_ interested in making the
build dependency on python go away, since it screws up my main
real-world test case for toybox (the aboriginal linux build, followed by
the native linux from scratch build).

But this isn't that. This is a complete rewrite of all the scripts in C,
doing pretty much what the current ones do. The Linux kernel developers
talk about "small patches that do one thing", and have sometimes used a
"trail of breadcrumbs" analogy. (I think Al Viro had a good explanation,
but haven't got net access right now.) The problem with a giant
take-it-or-leave-it lump that re-architects the an existing subsystem in
one go is that there's nothing you can bisect through if something
breaks later. If you try to track down a bug, debugging hits the
megapatch and then stops. You can't do the "it used to work, now it
doesn't, here's what changed" thing because _everything_ changed. The
fact it used to work is irrelevant.

That's not necessarily a showstopper, but it is a downside.

The build system has a design issues, as mentioned above. It doesn't
currently do what I want it to do. But this doesn't address any design
issues other than "the python dependency is inconvenient". And I'm not
sure the new design is an improvement: it manages to be a code
consolidation without being a code cleanup.

If we're going to redo how the build works and replace scripts/*.sh with
a single C file, I'd move it to the top level directory and call it
something like "generate.c" (since it creates generated/*), and then
clean out the scripts directory and possibly remove it. Move the
remaining stub script text back into the Makefile. End result: less
complexity. (I'd also want the generate.c file to be something I can
easily understand, which the current one isn't, but that could easily be
my fault. Bott times I read it I was pretty tired at the end of a long
day.)

By the way, the reason I keep getting hung up on the backlog is due to
architectural issues, like "three different dirtree() variants", "should
whoami be a wrapper or a separate command", and "is mentioning any
feature test macros for the headers a good idea or should I just have a
policy of never #defining _any_ of them and declaring build environments
that require them broken".

I also have pending cleanups: I haven't been able to add the extra ls
options so aboriginal can use it, or add -f to tail. Let alone doing
sed, mount, or converting nbd-client.c...

I haven't been able to focus on this because there's been an endless
stream of new code to review, introducing more architectural issues. I
am trying to keep complexity down, which means that the new pieces I add
to the puzzle have to be _right_. I feel really _bad_ about not merging
the code you're submitting, but I also have to look through Daniel
Walker's mode parser...

Rob
--
GNU/Linux isn't: Linux=GPLv2, GNU=GPLv3+, they can't share code.
Either it's "mere aggregation", or a license violation. Pick one.
Georgi Chorbadzhiyski
2012-03-21 22:36:32 UTC
Permalink
Post by Rob Landley
1) Remember when Daniel Walter did groups and whoami as reskins of id?
The current infrastructure supports aliases (two commands with the same
behavior), but does _not_ support two commands in the same file with
different main() functions and help text.
I think the current scheme of using different files for different
commands is extra easy to understand. Adding two or more commands
in a single C file is only asking for trouble. If there is shared functionality
it can easily be put in lib/blah.{c,h} and used by both commands.
Post by Rob Landley
2) I'm trying to add the nbd-client.c I wrote back in 2010, and with
config NBD_CLIENT the build is trying to include "nbd.c".
Using genconf.c and toys/nbd_client.c (the dash is problematic) results
in "working" nbd_client. Granted there should be a way for a command
to have dash in the name, so using "all-in" C file brings nothing
extra here (except that nbd_client.c builds and acts like a toy).
Post by Rob Landley
3) I want to combine the help text and usage: lines, as explained here
earlier.
Could you elaborate more on that, I can't find anything in the ML from
the last month or so. I searched mails for "usage" and "help". It seems
I have missed that.
Post by Rob Landley
4) I'd like to be able to build individual commands without necessarily
going through the config infrastructure. (That way I could build the
toybox sed/sort/shell commands to run the rest of the build even if the
host tools are strange, but it's also nice to be able to build a
non-swiss-army-knife version fo the binaries. I did "make individual"
for busybox long ago for similar reasons.)
It seems this depends only on generated/config.h. Should be easy to use
dummy config.h with only the needed command enabled and then copy resulting
toybox binary as generated/CMD.
Post by Rob Landley
5) I was hoping the config-parsing infrastructure I made could also act
as the start of a kconfig replacement. If the help text merging bits
have to understand the kconfig language thoroughly enough to parse "XXX
&& YYY" style dependencies (and without dependency tracking the rules on
what's an option to a command and what's a new command get REALLY
brittle in the presence of a single file with multiple commands that can
be named "pivot_root" and "newfs_msdos" and so on.)
Thats good but seeing how big kconfig is, it is no small task.
Post by Rob Landley
Merging this patch means I abandoning the (admittedly unfinished) work
I've done towards those goals, which would be fine if it addressed any
of them, but it doesn't.
It replaces it with a build infrastructure I've read through twice and
still not quite wrapped my head around yet (your buffer infrastructure
is doing what again: seems kind a overkill...). It brings back badig.h
associations to the old busybox "makedep" command. Its primary goal
seems to be making the project compile faster, which isn't exactly a
priority for me. (The current build doesn't even support smp
compilation, that's lower-hanging fruit than rewriting the shell scripts
in C.)
My primary goal was to simplify the build. Admittedly genconf.c can
be smaller but since it does couple of things intermixed it probably looks
scarier that it really is. The buffer infrastructure is very simple auto
growing string buffer.

If the parsing for different functionalities is separated it'll be
very easy to follow.

getconf.c does this:
1. Parse toys files and build generated/globals.h
2. Parse toys files and build generated/newtoys.h
3. Parse toys files and build generated/Config.in
4. Parse Config.in and generated/Config.in and build generated/help.h
5. Parse .config and build generated/config.h and generated/build_files

Version 2 that I have send should be easier to read:
https://github.com/gfto/toybox/commit/0a68d9cbdb77426d3383af92542e4ea0c62a102d#diff-3
Post by Rob Landley
If you'd done a patch that was _just_ changing the help text and leaving
the rest of it alone, I'd already have merged it and worried about
fixing up the rest I need later. I am _very_ interested in making the
build dependency on python go away, since it screws up my main
real-world test case for toybox (the aboriginal linux build, followed by
the native linux from scratch build).
I can easily leave only this functionality but it will not simplify the
build, it'll just replace python dependency with small C program and as
long as a C program have to be made why don't we do the other things
that current mix of shell, sed, sort, tr, etc does. It can be done step
by step of course.
Post by Rob Landley
But this isn't that. This is a complete rewrite of all the scripts in C,
doing pretty much what the current ones do. The Linux kernel developers
talk about "small patches that do one thing", and have sometimes used a
"trail of breadcrumbs" analogy. (I think Al Viro had a good explanation,
but haven't got net access right now.) The problem with a giant
take-it-or-leave-it lump that re-architects the an existing subsystem in
one go is that there's nothing you can bisect through if something
breaks later. If you try to track down a bug, debugging hits the
megapatch and then stops. You can't do the "it used to work, now it
doesn't, here's what changed" thing because _everything_ changed. The
fact it used to work is irrelevant.
I can replace each step in a different patch and if that is what will
help to get genconf to be merged I'll find the time to do it.
Post by Rob Landley
That's not necessarily a showstopper, but it is a downside.
The build system has a design issues, as mentioned above. It doesn't
currently do what I want it to do. But this doesn't address any design
issues other than "the python dependency is inconvenient". And I'm not
sure the new design is an improvement: it manages to be a code
consolidation without being a code cleanup.
The cleanup was the part where bunch of tools was replaced with one for
the specific need. A nice side effect was the build speedup (not important),
the ability to build toybox with just shell and C compiler.
Post by Rob Landley
If we're going to redo how the build works and replace scripts/*.sh with
a single C file, I'd move it to the top level directory and call it
something like "generate.c" (since it creates generated/*), and then
clean out the scripts directory and possibly remove it. Move the
remaining stub script text back into the Makefile. End result: less
complexity. (I'd also want the generate.c file to be something I can
easily understand, which the current one isn't, but that could easily be
my fault. Bott times I read it I was pretty tired at the end of a long
day.)
The naming is not a problem let it be generate.c. I left shell scripts
because I thought that they are there in order for build to not depend on
make. Put .config file in the root, run scripts/make.sh and you have toybox
binary.
Post by Rob Landley
By the way, the reason I keep getting hung up on the backlog is due to
architectural issues, like "three different dirtree() variants", "should
whoami be a wrapper or a separate command", and "is mentioning any
feature test macros for the headers a good idea or should I just have a
policy of never #defining _any_ of them and declaring build environments
that require them broken".
There a two ways here, merge and fix later (which will happen someday,
/the fix part/ possibly, may be not...), or sit on top of a pile and try
to merge it "when it is ready (tm)". It is up to you. I would merge everything
and cleanup later (or never :-) but that is just me.
Post by Rob Landley
I also have pending cleanups: I haven't been able to add the extra ls
options so aboriginal can use it, or add -f to tail. Let alone doing
sed, mount, or converting nbd-client.c...
IMHO you should write new code since this brings toybox closer 1.0
If we try to make every new toy close to perfect there will never be
enough time. And with new code there will be new issues with build system,
lib/, portability, etc. but the code will live in the tree and if possible
cleanups, feature additions, wish lists are documented perhaps somebody will
fix them. It doesn't have to be you.
Post by Rob Landley
I haven't been able to focus on this because there's been an endless
stream of new code to review, introducing more architectural issues. I
am trying to keep complexity down, which means that the new pieces I add
to the puzzle have to be _right_. I feel really _bad_ about not merging
the code you're submitting, but I also have to look through Daniel
Walker's mode parser...
I have slowed down now, the easy stuff is mostly done /and I'm good at
easy/, it would be great if I can do something to ease your burden.
--
Georgi Chorbadzhiyski
http://georgi.unixsol.org/
Georgi Chorbadzhiyski
2012-03-21 22:36:32 UTC
Permalink
Post by Rob Landley
1) Remember when Daniel Walter did groups and whoami as reskins of id?
The current infrastructure supports aliases (two commands with the same
behavior), but does _not_ support two commands in the same file with
different main() functions and help text.
I think the current scheme of using different files for different
commands is extra easy to understand. Adding two or more commands
in a single C file is only asking for trouble. If there is shared functionality
it can easily be put in lib/blah.{c,h} and used by both commands.
Post by Rob Landley
2) I'm trying to add the nbd-client.c I wrote back in 2010, and with
config NBD_CLIENT the build is trying to include "nbd.c".
Using genconf.c and toys/nbd_client.c (the dash is problematic) results
in "working" nbd_client. Granted there should be a way for a command
to have dash in the name, so using "all-in" C file brings nothing
extra here (except that nbd_client.c builds and acts like a toy).
Post by Rob Landley
3) I want to combine the help text and usage: lines, as explained here
earlier.
Could you elaborate more on that, I can't find anything in the ML from
the last month or so. I searched mails for "usage" and "help". It seems
I have missed that.
Post by Rob Landley
4) I'd like to be able to build individual commands without necessarily
going through the config infrastructure. (That way I could build the
toybox sed/sort/shell commands to run the rest of the build even if the
host tools are strange, but it's also nice to be able to build a
non-swiss-army-knife version fo the binaries. I did "make individual"
for busybox long ago for similar reasons.)
It seems this depends only on generated/config.h. Should be easy to use
dummy config.h with only the needed command enabled and then copy resulting
toybox binary as generated/CMD.
Post by Rob Landley
5) I was hoping the config-parsing infrastructure I made could also act
as the start of a kconfig replacement. If the help text merging bits
have to understand the kconfig language thoroughly enough to parse "XXX
&& YYY" style dependencies (and without dependency tracking the rules on
what's an option to a command and what's a new command get REALLY
brittle in the presence of a single file with multiple commands that can
be named "pivot_root" and "newfs_msdos" and so on.)
Thats good but seeing how big kconfig is, it is no small task.
Post by Rob Landley
Merging this patch means I abandoning the (admittedly unfinished) work
I've done towards those goals, which would be fine if it addressed any
of them, but it doesn't.
It replaces it with a build infrastructure I've read through twice and
still not quite wrapped my head around yet (your buffer infrastructure
is doing what again: seems kind a overkill...). It brings back badig.h
associations to the old busybox "makedep" command. Its primary goal
seems to be making the project compile faster, which isn't exactly a
priority for me. (The current build doesn't even support smp
compilation, that's lower-hanging fruit than rewriting the shell scripts
in C.)
My primary goal was to simplify the build. Admittedly genconf.c can
be smaller but since it does couple of things intermixed it probably looks
scarier that it really is. The buffer infrastructure is very simple auto
growing string buffer.

If the parsing for different functionalities is separated it'll be
very easy to follow.

getconf.c does this:
1. Parse toys files and build generated/globals.h
2. Parse toys files and build generated/newtoys.h
3. Parse toys files and build generated/Config.in
4. Parse Config.in and generated/Config.in and build generated/help.h
5. Parse .config and build generated/config.h and generated/build_files

Version 2 that I have send should be easier to read:
https://github.com/gfto/toybox/commit/0a68d9cbdb77426d3383af92542e4ea0c62a102d#diff-3
Post by Rob Landley
If you'd done a patch that was _just_ changing the help text and leaving
the rest of it alone, I'd already have merged it and worried about
fixing up the rest I need later. I am _very_ interested in making the
build dependency on python go away, since it screws up my main
real-world test case for toybox (the aboriginal linux build, followed by
the native linux from scratch build).
I can easily leave only this functionality but it will not simplify the
build, it'll just replace python dependency with small C program and as
long as a C program have to be made why don't we do the other things
that current mix of shell, sed, sort, tr, etc does. It can be done step
by step of course.
Post by Rob Landley
But this isn't that. This is a complete rewrite of all the scripts in C,
doing pretty much what the current ones do. The Linux kernel developers
talk about "small patches that do one thing", and have sometimes used a
"trail of breadcrumbs" analogy. (I think Al Viro had a good explanation,
but haven't got net access right now.) The problem with a giant
take-it-or-leave-it lump that re-architects the an existing subsystem in
one go is that there's nothing you can bisect through if something
breaks later. If you try to track down a bug, debugging hits the
megapatch and then stops. You can't do the "it used to work, now it
doesn't, here's what changed" thing because _everything_ changed. The
fact it used to work is irrelevant.
I can replace each step in a different patch and if that is what will
help to get genconf to be merged I'll find the time to do it.
Post by Rob Landley
That's not necessarily a showstopper, but it is a downside.
The build system has a design issues, as mentioned above. It doesn't
currently do what I want it to do. But this doesn't address any design
issues other than "the python dependency is inconvenient". And I'm not
sure the new design is an improvement: it manages to be a code
consolidation without being a code cleanup.
The cleanup was the part where bunch of tools was replaced with one for
the specific need. A nice side effect was the build speedup (not important),
the ability to build toybox with just shell and C compiler.
Post by Rob Landley
If we're going to redo how the build works and replace scripts/*.sh with
a single C file, I'd move it to the top level directory and call it
something like "generate.c" (since it creates generated/*), and then
clean out the scripts directory and possibly remove it. Move the
remaining stub script text back into the Makefile. End result: less
complexity. (I'd also want the generate.c file to be something I can
easily understand, which the current one isn't, but that could easily be
my fault. Bott times I read it I was pretty tired at the end of a long
day.)
The naming is not a problem let it be generate.c. I left shell scripts
because I thought that they are there in order for build to not depend on
make. Put .config file in the root, run scripts/make.sh and you have toybox
binary.
Post by Rob Landley
By the way, the reason I keep getting hung up on the backlog is due to
architectural issues, like "three different dirtree() variants", "should
whoami be a wrapper or a separate command", and "is mentioning any
feature test macros for the headers a good idea or should I just have a
policy of never #defining _any_ of them and declaring build environments
that require them broken".
There a two ways here, merge and fix later (which will happen someday,
/the fix part/ possibly, may be not...), or sit on top of a pile and try
to merge it "when it is ready (tm)". It is up to you. I would merge everything
and cleanup later (or never :-) but that is just me.
Post by Rob Landley
I also have pending cleanups: I haven't been able to add the extra ls
options so aboriginal can use it, or add -f to tail. Let alone doing
sed, mount, or converting nbd-client.c...
IMHO you should write new code since this brings toybox closer 1.0
If we try to make every new toy close to perfect there will never be
enough time. And with new code there will be new issues with build system,
lib/, portability, etc. but the code will live in the tree and if possible
cleanups, feature additions, wish lists are documented perhaps somebody will
fix them. It doesn't have to be you.
Post by Rob Landley
I haven't been able to focus on this because there's been an endless
stream of new code to review, introducing more architectural issues. I
am trying to keep complexity down, which means that the new pieces I add
to the puzzle have to be _right_. I feel really _bad_ about not merging
the code you're submitting, but I also have to look through Daniel
Walker's mode parser...
I have slowed down now, the easy stuff is mostly done /and I'm good at
easy/, it would be great if I can do something to ease your burden.
--
Georgi Chorbadzhiyski
http://georgi.unixsol.org/
Rob Landley
2012-03-19 13:35:16 UTC
Permalink
Post by Georgi Chorbadzhiyski
Post by Rob Landley
Post by Georgi Chorbadzhiyski
Post by Rob Landley
Post by Georgi Chorbadzhiyski
+void generate_headers(int argc, char **argv) {
+ // Parse toys
+ strbuf_add(&newtoys_h, "NEWTOY(toybox, NULL, 0)\n");
+ strbuf_add(&globals_union, "extern union global_union {\n");
It matters that these be in alphabetical order (for the binary search),
and we can't depend on the filesystem doing it for us. Where are you
sorting this?
I didn't sort them. I depend on the shell doing the right thing. The program
is called like this generated/genconf toys/*.c so it should be OK. Of course
the parameters can be sorted but it seemed unnecessary since the shell is
doing that for us.
Actually the shell isn't. The filesystem is. (I've hit this one
before, it's not reliable.)
Also, sorting the .c files isn't sufficient, you have to sort the
_output_ array, because one .c file can produce more than one array
entry (sh and toysh, netcat and nc...)
I'll send an updated patch tomorrow.
The C command I was working on had several goals:

1) Remember when Daniel Walter did groups and whoami as reskins of id?
The current infrastructure supports aliases (two commands with the same
behavior), but does _not_ support two commands in the same file with
different main() functions and help text.

2) I'm trying to add the nbd-client.c I wrote back in 2010, and with
config NBD_CLIENT the build is trying to include "nbd.c".

3) I want to combine the help text and usage: lines, as explained here
earlier.

4) I'd like to be able to build individual commands without necessarily
going through the config infrastructure. (That way I could build the
toybox sed/sort/shell commands to run the rest of the build even if the
host tools are strange, but it's also nice to be able to build a
non-swiss-army-knife version fo the binaries. I did "make individual"
for busybox long ago for similar reasons.)

5) I was hoping the config-parsing infrastructure I made could also act
as the start of a kconfig replacement. If the help text merging bits
have to understand the kconfig language thoroughly enough to parse "XXX
&& YYY" style dependencies (and without dependency tracking the rules on
what's an option to a command and what's a new command get REALLY
brittle in the presence of a single file with multiple commands that can
be named "pivot_root" and "newfs_msdos" and so on.)

Merging this patch means I abandoning the (admittedly unfinished) work
I've done towards those goals, which would be fine if it addressed any
of them, but it doesn't.

It replaces it with a build infrastructure I've read through twice and
still not quite wrapped my head around yet (your buffer infrastructure
is doing what again: seems kind a overkill...). It brings back bad
associations to the old busybox "makedep" command. Its primary goal
seems to be making the project compile faster, which isn't exactly a
priority for me. (The current build doesn't even support smp
compilation, that's lower-hanging fruit than rewriting the shell scripts
in C.)

If you'd done a patch that was _just_ changing the help text and leaving
the rest of it alone, I'd already have merged it and worried about
fixing up the rest I need later. I am _very_ interested in making the
build dependency on python go away, since it screws up my main
real-world test case for toybox (the aboriginal linux build, followed by
the native linux from scratch build).

But this isn't that. This is a complete rewrite of all the scripts in C,
doing pretty much what the current ones do. The Linux kernel developers
talk about "small patches that do one thing", and have sometimes used a
"trail of breadcrumbs" analogy. (I think Al Viro had a good explanation,
but haven't got net access right now.) The problem with a giant
take-it-or-leave-it lump that re-architects the an existing subsystem in
one go is that there's nothing you can bisect through if something
breaks later. If you try to track down a bug, debugging hits the
megapatch and then stops. You can't do the "it used to work, now it
doesn't, here's what changed" thing because _everything_ changed. The
fact it used to work is irrelevant.

That's not necessarily a showstopper, but it is a downside.

The build system has a design issues, as mentioned above. It doesn't
currently do what I want it to do. But this doesn't address any design
issues other than "the python dependency is inconvenient". And I'm not
sure the new design is an improvement: it manages to be a code
consolidation without being a code cleanup.

If we're going to redo how the build works and replace scripts/*.sh with
a single C file, I'd move it to the top level directory and call it
something like "generate.c" (since it creates generated/*), and then
clean out the scripts directory and possibly remove it. Move the
remaining stub script text back into the Makefile. End result: less
complexity. (I'd also want the generate.c file to be something I can
easily understand, which the current one isn't, but that could easily be
my fault. Bott times I read it I was pretty tired at the end of a long
day.)

By the way, the reason I keep getting hung up on the backlog is due to
architectural issues, like "three different dirtree() variants", "should
whoami be a wrapper or a separate command", and "is mentioning any
feature test macros for the headers a good idea or should I just have a
policy of never #defining _any_ of them and declaring build environments
that require them broken".

I also have pending cleanups: I haven't been able to add the extra ls
options so aboriginal can use it, or add -f to tail. Let alone doing
sed, mount, or converting nbd-client.c...

I haven't been able to focus on this because there's been an endless
stream of new code to review, introducing more architectural issues. I
am trying to keep complexity down, which means that the new pieces I add
to the puzzle have to be _right_. I feel really _bad_ about not merging
the code you're submitting, but I also have to look through Daniel
Walker's mode parser...

Rob
--
GNU/Linux isn't: Linux=GPLv2, GNU=GPLv3+, they can't share code.
Either it's "mere aggregation", or a license violation. Pick one.
Georgi Chorbadzhiyski
2012-03-18 21:17:13 UTC
Permalink
Post by Rob Landley
Post by Georgi Chorbadzhiyski
Post by Rob Landley
Post by Georgi Chorbadzhiyski
+void generate_headers(int argc, char **argv) {
+ // Parse toys
+ strbuf_add(&newtoys_h, "NEWTOY(toybox, NULL, 0)\n");
+ strbuf_add(&globals_union, "extern union global_union {\n");
It matters that these be in alphabetical order (for the binary search),
and we can't depend on the filesystem doing it for us. Where are you
sorting this?
I didn't sort them. I depend on the shell doing the right thing. The program
is called like this generated/genconf toys/*.c so it should be OK. Of course
the parameters can be sorted but it seemed unnecessary since the shell is
doing that for us.
Actually the shell isn't. The filesystem is. (I've hit this one
before, it's not reliable.)
Also, sorting the .c files isn't sufficient, you have to sort the
_output_ array, because one .c file can produce more than one array
entry (sh and toysh, netcat and nc...)
I'll send an updated patch tomorrow.
--
Georgi Chorbadzhiyski
http://georgi.unixsol.org/
Rob Landley
2012-03-18 16:55:06 UTC
Permalink
Post by Georgi Chorbadzhiyski
Post by Rob Landley
Post by Georgi Chorbadzhiyski
This patch removes all external tools from the build system. Now
everything in generated/ directory is build with single program.
The effect is that the build no longer requires sed, tr, sort and
python (for rebuilding the help texts). There is no need to ship
generated/help.h since it can be rebuild on the host without python.
Cool!
Generating help.h in C (and eliminating the need for python) is
something I've been meaning to do for a while. But building the list of
C files to compile... in C... seems unnecessarily brittle somehow.
No more than the sed usage (which I was unable to simplify and this is
was drove me to make the C version).
I'm a bit biased because I wrote the busybox sed implementation. (And
string handling in C is pretty inherently brittle...)
Post by Georgi Chorbadzhiyski
Post by Rob Landley
Post by Georgi Chorbadzhiyski
+void generate_headers(int argc, char **argv) {
+ // Parse toys
+ strbuf_add(&newtoys_h, "NEWTOY(toybox, NULL, 0)\n");
+ strbuf_add(&globals_union, "extern union global_union {\n");
It matters that these be in alphabetical order (for the binary search),
and we can't depend on the filesystem doing it for us. Where are you
sorting this?
I didn't sort them. I depend on the shell doing the right thing. The program
is called like this generated/genconf toys/*.c so it should be OK. Of course
the parameters can be sorted but it seemed unnecessary since the shell is
doing that for us.
Actually the shell isn't. The filesystem is. (I've hit this one
before, it's not reliable.)

Also, sorting the .c files isn't sufficient, you have to sort the
_output_ array, because one .c file can produce more than one array
entry (sh and toysh, netcat and nc...)

Rob
--
GNU/Linux isn't: Linux=GPLv2, GNU=GPLv3+, they can't share code.
Either it's "mere aggregation", or a license violation. Pick one.
Georgi Chorbadzhiyski
2012-03-18 16:14:19 UTC
Permalink
Post by Rob Landley
Post by Georgi Chorbadzhiyski
This patch removes all external tools from the build system. Now
everything in generated/ directory is build with single program.
The effect is that the build no longer requires sed, tr, sort and
python (for rebuilding the help texts). There is no need to ship
generated/help.h since it can be rebuild on the host without python.
Cool!
Generating help.h in C (and eliminating the need for python) is
something I've been meaning to do for a while. But building the list of
C files to compile... in C... seems unnecessarily brittle somehow.
No more than the sed usage (which I was unable to simplify and this is
was drove me to make the C version).
Post by Rob Landley
Post by Georgi Chorbadzhiyski
+__attribute__ ((format(printf, 2, 3)))
+void strbuf_printf(struct strbuf *s, const char *fmt, ...) {
This is a purely local function, the __attribute__ tag is overkill.
It is just a habit I've picked when adding new printf style varargs
functions.
Post by Rob Landley
Post by Georgi Chorbadzhiyski
+void generate_headers(int argc, char **argv) {
+ // Parse toys
+ strbuf_add(&newtoys_h, "NEWTOY(toybox, NULL, 0)\n");
+ strbuf_add(&globals_union, "extern union global_union {\n");
It matters that these be in alphabetical order (for the binary search),
and we can't depend on the filesystem doing it for us. Where are you
sorting this?
I didn't sort them. I depend on the shell doing the right thing. The program
is called like this generated/genconf toys/*.c so it should be OK. Of course
the parameters can be sorted but it seemed unnecessary since the shell is
doing that for us.
--
Georgi Chorbadzhiyski
http://georgi.unixsol.org/
Georgi Chorbadzhiyski
2012-03-16 22:05:06 UTC
Permalink
This patch removes all external tools from the build system. Now
everything in generated/ directory is build with single program.
The effect is that the build no longer requires sed, tr, sort and
python (for rebuilding the help texts). There is no need to ship
generated/help.h since it can be rebuild on the host without python.

There is also a nice speedup. On my machine building allyesconfig
is ~1.8 seconds faster, it goes down from 11.3s to 0m9.5s.
---
Makefile | 7 +-
scripts/config2help.py | 54 ---------
scripts/config2help.sh | 54 ---------
scripts/genconf.c | 309 ++++++++++++++++++++++++++++++++++++++++++++++++
scripts/genconfig.sh | 10 ++-
scripts/make.sh | 88 +--------------
6 files changed, 324 insertions(+), 198 deletions(-)
delete mode 100755 scripts/config2help.py
delete mode 100755 scripts/config2help.sh
create mode 100644 scripts/genconf.c

diff --git a/Makefile b/Makefile
index 7079f82..4aec0ff 100644
--- a/Makefile
+++ b/Makefile
@@ -17,6 +17,9 @@ generated/Config.in: toys/*.c scripts/genconfig.sh

HOSTCC?=cc

+.config:
+ $(MAKE) defconfig
+
# Development targets
baseline: toybox_unstripped
@cp toybox_unstripped toybox_old
@@ -42,10 +45,10 @@ uninstall:
clean::
rm -rf toybox toybox_unstripped generated/config.h generated/Config.in \
generated/newtoys.h generated/globals.h instlist testdir \
- generated/Config.probed
+ generated/Config.probed generated/help.h generated/build_files

distclean: clean
- rm -f toybox_old .config* generated/help.h
+ rm -f toybox_old .config* generated/genconf

test: tests

diff --git a/scripts/config2help.py b/scripts/config2help.py
deleted file mode 100755
index 2573d08..0000000
--- a/scripts/config2help.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/python
-
-import os,sys
-
-def zapquotes(str):
- if str[0]=='"': str = str[1:str.rfind('"')]
- return str
-
-def escapequotes(str):
- return str.strip().replace("\\","\\\\").replace('"','\\"')
-
-helplen = morelines = 0
-out = sys.stdout
-
-def readfile(filename):
- global helplen, morelines
- #sys.stderr.write("Reading %s\n" % filename)
- try:
- lines = open(filename).read().split("\n")
- except IOError:
- sys.stderr.write("File %s missing\n" % filename)
- return
- config = None
- description = None
- for i in lines:
- if helplen:
- i = i.expandtabs()
- if not len(i) or i[:helplen].isspace():
- if morelines: out.write('\\n')
- morelines = 1
- out.write(escapequotes(i))
- continue
- else:
- helplen = morelines = 0
- out.write('"\n')
-
- words = i.strip().split(None,1)
- if not len(words): continue
-
- if words[0] in ("config", "menuconfig"):
- config = words[1]
- description = ""
- elif words[0] in ("bool", "boolean", "tristate", "string", "hex", "int"):
- if len(words)>1: description = zapquotes(words[1])
- elif words[0]=="prompt":
- description = htmlescape(zapquotes(words[1]))
- elif words[0] in ("help", "---help---"):
- out.write('#define help_%s "' % config.lower())
- helplen = len(i[:i.find(words[0])].expandtabs())
- elif words[0] == "source": readfile(zapquotes(words[1]))
- elif words[0] in ("default","depends", "select", "if", "endif", "#", "comment", "menu", "endmenu"): pass
-
-readfile(sys.argv[1])
-if helplen: out.write('"\n')
diff --git a/scripts/config2help.sh b/scripts/config2help.sh
deleted file mode 100755
index 63d6c6f..0000000
--- a/scripts/config2help.sh
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/bin/bash -e
-
-function firstmajor() {
- declare -i n j=$1
- test $j -gt 0 || return -1
- for j in $@; do
- if [ $j -gt $1 ]; then
- echo $j
- return 0
- fi
- done
- return 1
-}; export -f firstmajor
-
-function print_h() {
- declare -i i c=$2 s=$3 e=$4
- local str="$(echo "$1" | head -n$c | tail -n1 | sed -e "s,config[\t ]*,,")"
- echo -n "#define help_"$str" \"" | tr [A-Z] [a-z]
- echo -n "$1\\n" | head -n$e | tail -n$[e-s+1] | sed -e "s,\$,\r," | tr \\n\\r n\\
- echo \"
-}; export -f print_h
-
-file="$1"
-if test "$0" != "bash"; then
- if test -r "$file"; then
-# echo "$file..." >&2
- filetxt="$(sed -e "s,^[\t ]*,," -e "s,\([\"\\\\]\),\\\\\\1,g" "$file")"
- helps=$(echo "$filetxt" | egrep -ne "^help *" -e "^---help--- *" | cut -d\: -f1)
- configs=$(echo "$filetxt" | egrep -ne "^config *" | cut -d\: -f1)
- endmenus=$(echo "$filetxt" | egrep -ne "^endmenu *" | cut -d\: -f1)
- let last=$(echo "$filetxt" | wc -l)+2
-
- declare -i i c s e
- for i in $configs; do
-# echo -n "c:$i" >&2
- s=$(firstmajor $i $helps)
- test $s -gt 0
- e=$(firstmajor $s $configs || firstmajor $s $endmenus $last)
- let s++ e-=2
- test $e -ge $s
-# echo " s:$s e:$e" >&2
- print_h "$filetxt" $i $s $e
- done
- for fle in $(cat "$file" | egrep -e "^[ \t]*source " | sed -e "s,^[ \t]*source *\(.*\),\\1,"); do
- $0 $fle
- done
- else
- echo
- echo "USAGE EXAMPLE: $(basename $0) Config.in > generated/help.h"
- echo
- false
- fi | sed -e "s,\\\\n\\\\n\"$,\\\\n\","
-fi
-
diff --git a/scripts/genconf.c b/scripts/genconf.c
new file mode 100644
index 0000000..8346dee
--- /dev/null
+++ b/scripts/genconf.c
@@ -0,0 +1,309 @@
+/* vi: set ts=4 :*/
+/*
+ * Generate toybox generated/{Config.in,config.h,globals.h,newtoys.h,help.h}
+ *
+ * Copyright 2012 Georgi Chorbadzhiyski <georgi at unixsol.org>
+ */
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <unistd.h>
+#include <errno.h>
+
+struct strbuf {
+ unsigned int len;
+ unsigned int nchars;
+ char *data;
+};
+
+struct strbuf newtoys_h;
+struct strbuf config_h;
+struct strbuf config_in;
+struct strbuf help_h;
+struct strbuf globals_h;
+struct strbuf globals_union;
+struct strbuf build_files;
+
+int is_toy;
+int in_config;
+int in_help;
+int in_globals;
+
+char *line = NULL;
+size_t linelen;
+char *pos, *toyname, *conf;
+int i;
+char tmp[1024];
+
+
+FILE *f;
+
+FILE *xfopen(char *filename, char *mode) {
+ FILE *fh = fopen(filename, mode);
+ if (!fh) {
+ fprintf(stderr, "fopen(%s, %s): %s\n", filename, mode, strerror(errno));
+ exit(-1);
+ }
+ return fh;
+}
+
+void strbuf_add(struct strbuf *s, char *text) {
+ int len = strlen(text);
+ if (!len)
+ return;
+ if (!s->data) {
+ s->nchars = 1024;
+ s->data = calloc(1, s->nchars);
+ }
+ s->len += len;
+ if (s->len >= s->nchars) {
+ s->nchars += s->len + 1;
+ s->data = realloc(s->data, s->nchars);
+ if (!s->data) {
+ fprintf(stderr, "realloc(%d): %s\n", s->nchars, strerror(errno));
+ exit(-1);
+ }
+ }
+ strcat(s->data, text);
+}
+
+__attribute__ ((format(printf, 2, 3)))
+void strbuf_printf(struct strbuf *s, const char *fmt, ...) {
+ char line[4096];
+
+ va_list args;
+ va_start(args, fmt);
+ vsnprintf(line, sizeof(line) - 1, fmt, args);
+ va_end(args);
+
+ line[sizeof(line) - 1] = '\0';
+ strbuf_add(s, line);
+}
+
+void strbuf_add_escaped(struct strbuf *s, char *text) {
+ int len = strlen(text), i;
+ if (len == 0)
+ strbuf_add(s, "\\n");
+ for (i = 0; i < len; i++) {
+ char c[2] = { text[i], 0 };
+ switch (text[i]) {
+ case '\n': strbuf_add(s, "\\n"); break;
+ case '\t': strbuf_add(s, " "); break;
+ case '\\': strbuf_add(s, "\\\\"); break;
+ case '"' : strbuf_add(s, "\\\""); break;
+ default : strbuf_add(s, c); break;
+ }
+ }
+}
+
+void strbuf_dump(struct strbuf *s, char *filename, char *mode) {
+ FILE *fh;
+ if (!s->data) {
+ unlink(filename);
+ return;
+ }
+ fh = xfopen(filename, mode);
+ fwrite(s->data, s->len, 1, fh);
+ fclose(fh);
+ free(s->data);
+}
+
+void parse_help(char *filename) {
+ int i, nitems = 0;
+ char *config_name = NULL;
+
+ f = xfopen(filename, "r");
+ while (getline(&line, &linelen, f) > 0) {
+ if (line[0] == '#')
+ continue;
+ if (strncmp(line, "config ", 7) == 0) {
+ in_config = 1;
+ in_help = 0;
+ free(config_name);
+ config_name = strdup(line + 7);
+ int config_len = strlen(config_name);
+ for (i = 0; i < config_len; i++) {
+ config_name[i] = tolower((unsigned char)config_name[i]);
+ if (config_name[i] == '\n')
+ config_name[i] = '\0';
+ }
+ // Close already started help line
+ if (nitems)
+ strbuf_add(&help_h, "\"\n");
+ continue;
+ }
+ if (strncmp(line, "endmenu", 7) == 0) {
+ in_config = 0;
+ in_help = 0;
+ }
+ if (!in_config)
+ continue;
+
+ if (!in_help) {
+ if (strncmp(line, "\thelp", 5) == 0 || strncmp(line, " help", 8) == 0) {
+ in_help = 1;
+ nitems++;
+ strbuf_printf(&help_h, "#define help_%s \"", config_name);
+ }
+ continue;
+ } else {
+ char *tmpline = line;
+ while (isspace(*tmpline))
+ tmpline++;
+ strbuf_add_escaped(&help_h, tmpline);
+ continue;
+ }
+ }
+ strbuf_add(&help_h, "\"\n");
+
+ fclose(f);
+
+ free(config_name);
+}
+
+void generate_headers(int argc, char **argv) {
+ // Parse toys
+ strbuf_add(&newtoys_h, "NEWTOY(toybox, NULL, 0)\n");
+ strbuf_add(&globals_union, "extern union global_union {\n");
+
+ for (i = 1; i < argc; i++) {
+ f = xfopen(argv[i], "r");
+
+ toyname = argv[i];
+ pos = strrchr(toyname, '/');
+ if (pos)
+ toyname = pos + 1; // Remove path
+ pos = strstr(toyname, ".c");
+ if (pos) // Remove extension
+ pos[0] = '\0';
+
+ strbuf_printf(&globals_h, "// %s.c\n\n", argv[i]);
+
+ while (getline(&line, &linelen, f) > 0) {
+ // Parse toys
+ if (strncmp(line, "USE_", 4) == 0) {
+ is_toy = 1;
+ strbuf_add(&newtoys_h, line);
+ continue;
+ }
+ if (!is_toy)
+ continue;
+
+ // Parse config
+ if (!in_config) {
+ if (strncmp(line, "config ", 7) == 0) {
+ in_config = 1;
+ strbuf_printf(&config_in, "# %s.c\n", argv[i]);
+ strbuf_add(&config_in, line);
+ continue;
+ }
+ } else {
+ if (strncmp(line, "*/", 2) == 0) {
+ strbuf_add(&config_in, "\n");
+ in_config = 0;
+ continue;
+ } else {
+ strbuf_add(&config_in, line);
+ }
+ }
+
+ // Parse globals
+ if (!in_globals) {
+ if (strncmp(line, "DEFINE_GLOBALS(", 15) == 0) {
+ strbuf_printf(&globals_h, "struct %s_data {\n", toyname);
+ strbuf_printf(&globals_union, "\tstruct %s_data %s;\n", toyname, toyname);
+ in_globals = 1;
+ continue;
+ }
+ } else {
+ if (strncmp(line, ")", 1) == 0) {
+ strbuf_add(&globals_h, "};\n");
+ in_globals = 0;
+ break;
+ } else {
+ strbuf_add(&globals_h, line);
+ }
+ }
+ }
+ fclose(f);
+ }
+
+ strbuf_add(&globals_union, "} this;\n");
+
+ strbuf_dump(&config_in, "generated/Config.in", "w");
+ strbuf_dump(&globals_h, "generated/globals.h", "w");
+ strbuf_dump(&globals_union, "generated/globals.h", "a");
+ strbuf_dump(&newtoys_h, "generated/newtoys.h", "w");
+}
+
+void generate_config_in() {
+ parse_help("Config.in");
+ parse_help("generated/Config.in");
+ strbuf_dump(&help_h, "generated/help.h", "w");
+}
+
+void parse_config() {
+ // Configu do not exists, create dummy .config
+ if (!access(".config", R_OK) == 0) {
+ f = xfopen(".config", "w");
+ fclose(f);
+ return;
+ }
+
+ // Parse .config
+ f = xfopen(".config", "r");
+ while (getline(&line, &linelen, f) > 0) {
+ if (strncmp(line, "CONFIG_", 7) == 0) {
+ // Config is ON
+ conf = line + 7;
+ pos = strchr(conf, '=');
+ if (pos) pos[0] = '\0';
+ strbuf_printf(&config_h, "#define CFG_%s 1\n", conf);
+ strbuf_printf(&config_h, "#define USE_%s(...) __VA_ARGS__\n", conf);
+ if (strncmp(conf, "TOYBOX_", 7) != 0) {
+ // Generate build_files
+ int config_len = strlen(conf);
+ for (i = 0; i < config_len; i++) {
+ conf[i] = tolower((unsigned char)conf[i]);
+ if (conf[i] == '\n' || conf[i] == ' ' || conf[i] == '=')
+ conf[i] = '\0';
+ }
+ snprintf(tmp, sizeof(tmp) - 1, "toys/%s.c", conf);
+ tmp[sizeof(tmp) - 1] = '\0';
+ if (access(tmp, R_OK) == 0)
+ strbuf_printf(&build_files, "%s\n", tmp);
+ }
+ continue;
+ }
+ if (strncmp(line, "# CONFIG_", 9) == 0) {
+ // Config is OFF
+ conf = line + 9;
+ pos = strchr(conf, ' ');
+ if (pos) pos[0] = '\0';
+ strbuf_printf(&config_h, "#define CFG_%s 0\n", conf);
+ strbuf_printf(&config_h, "#define USE_%s(...)\n", conf);
+ continue;
+ }
+ }
+ fclose(f);
+ strbuf_dump(&config_h, "generated/config.h", "w");
+ strbuf_dump(&build_files, "generated/build_files", "w");
+}
+
+
+int main(int argc, char **argv) {
+ if (argc < 2) {
+ fprintf(stderr, "Usage: %s toys/*.c\n", argv[0]);
+ exit(1);
+ }
+
+ generate_headers(argc, argv);
+ generate_config_in();
+ parse_config();
+
+ free(line);
+
+ return 0;
+}
diff --git a/scripts/genconfig.sh b/scripts/genconfig.sh
index f677954..c562e37 100755
--- a/scripts/genconfig.sh
+++ b/scripts/genconfig.sh
@@ -34,4 +34,12 @@ genconfig()
}

probeconfig > generated/Config.probed || rm generated/Config.probed
-genconfig > generated/Config.in || rm generated/Config.in
+
+if [ ! -x generated/genconf ]
+then
+ echo "Building generated/genconf"
+ $HOSTCC -ggdb -Wall -Wextra -Wno-sign-compare -I . -o generated/genconf scripts/genconf.c || exit 1
+fi
+
+echo "Extract configuration information from toys/*.c files..."
+generated/genconf toys/*.c || exit 1
diff --git a/scripts/make.sh b/scripts/make.sh
index f5bf0fd..e9afa68 100755
--- a/scripts/make.sh
+++ b/scripts/make.sh
@@ -4,95 +4,9 @@

source ./configure

-echo "Extract configuration information from toys/*.c files..."
scripts/genconfig.sh

-echo "Generate headers from toys/*.c..."
-
-# Create a list of all the applets toybox can provide. Note that the first
-# entry is out of order on purpose (the toybox multiplexer applet must be the
-# first element of the array). The rest must be sorted in alphabetical order
-# for fast binary search.
-
-function newtoys()
-{
- for i in toys/*.c
- do
- sed -n -e '1,/^config [A-Z]/s/^USE_/&/p' $i || exit 1
- done
-}
-echo "NEWTOY(toybox, NULL, 0)" > generated/newtoys.h
-newtoys | sed 's/\(.*TOY(\)\([^,]*\),\(.*\)/\2 \1\2,\3/' | sort -k 1,1 \
- | sed 's/[^ ]* //' >> generated/newtoys.h
-
-# Extract global structure definitions from toys/*.c
-
-function getglobals()
-{
- for i in toys/*.c
- do
- NAME="$(echo $i | sed 's at toys/\(.*\)\.c@\1@')"
-
- echo -e "// $i\n"
- sed -n -e '/^DEFINE_GLOBALS(/,/^)/b got;b;:got' \
- -e 's/^DEFINE_GLOBALS(/struct '"$NAME"'_data {/' \
- -e 's/^)/};/' -e 'p' $i
- done
-}
-
-GLOBSTRUCT="$(getglobals)"
-(
- echo "$GLOBSTRUCT"
- echo
- echo "extern union global_union {"
- echo "$GLOBSTRUCT" | sed -n 's/struct \(.*\)_data {/ struct \1_data \1;/p'
- echo "} this;"
-) > generated/globals.h
-
-# Only recreate generated/help.h if python is installed
-if [ ! -z "$(which python)" ] && [ ! -z "$(grep 'CONFIG_HELP=y' .config)" ]
-then
- echo "Extract help text from Config.in."
- scripts/config2help.py Config.in > generated/help.h || exit 1
-fi
-
-echo "Make generated/config.h from .config."
-
-# This long and roundabout sed invocation is to make old versions of sed happy.
-# New ones have '\n' so can replace one line with two without all the branches
-# and tedious mucking about with hold space.
-
-sed -n \
- -e 's/^# CONFIG_\(.*\) is not set.*/\1/' \
- -e 't notset' \
- -e 's/^CONFIG_\(.*\)=y.*/\1/' \
- -e 't isset' \
- -e 's/^CONFIG_\([^=]*\)=\(.*\)/#define CFG_\1 \2/p' \
- -e 'd' \
- -e ':notset' \
- -e 'h' \
- -e 's/.*/#define CFG_& 0/p' \
- -e 'g' \
- -e 's/.*/#define USE_&(...)/p' \
- -e 'd' \
- -e ':isset' \
- -e 'h' \
- -e 's/.*/#define CFG_& 1/p' \
- -e 'g' \
- -e 's/.*/#define USE_&(...) __VA_ARGS__/p' \
- .config > generated/config.h || exit 1
-
-# Extract a list of toys/*.c files to compile from the data in ".config" with
-# sed, sort, and tr:
-
-# 1) Grab the XXX part of all CONFIG_XXX entries, removing everything after the
-# second underline
-# 2) Sort the list, keeping only one of each entry.
-# 3) Convert to lower case.
-# 4) Remove toybox itself from the list (as that indicates global symbols).
-# 5) Add "toys/" prefix and ".c" suffix.
-
-TOYFILES=$(cat .config | sed -nre 's/^CONFIG_(.*)=y/\1/;t skip;b;:skip;s/_.*//;p' | sort -u | tr A-Z a-z | grep -v '^toybox$' | sed 's@\(.*\)@toys/\1.c@' )
+TOYFILES=$(cat generated/build_files 2>/dev/null)

echo "Compile toybox..."
--
1.7.5.1
Rob Landley
2012-03-17 02:57:45 UTC
Permalink
Post by Georgi Chorbadzhiyski
This patch removes all external tools from the build system. Now
everything in generated/ directory is build with single program.
The effect is that the build no longer requires sed, tr, sort and
python (for rebuilding the help texts). There is no need to ship
generated/help.h since it can be rebuild on the host without python.
Cool!

Generating help.h in C (and eliminating the need for python) is
something I've been meaning to do for a while. But building the list of
C files to compile... in C... seems unnecessarily brittle somehow.
Post by Georgi Chorbadzhiyski
There is also a nice speedup. On my machine building allyesconfig
is ~1.8 seconds faster, it goes down from 11.3s to 0m9.5s.
---
Makefile | 7 +-
scripts/config2help.py | 54 ---------
scripts/config2help.sh | 54 ---------
scripts/genconf.c | 309 ++++++++++++++++++++++++++++++++++++++++++++++++
scripts/genconfig.sh | 10 ++-
scripts/make.sh | 88 +--------------
6 files changed, 324 insertions(+), 198 deletions(-)
delete mode 100755 scripts/config2help.py
delete mode 100755 scripts/config2help.sh
create mode 100644 scripts/genconf.c
diff --git a/Makefile b/Makefile
index 7079f82..4aec0ff 100644
--- a/Makefile
+++ b/Makefile
@@ -17,6 +17,9 @@ generated/Config.in: toys/*.c scripts/genconfig.sh
HOSTCC?=cc
+ $(MAKE) defconfig
+
# Development targets
baseline: toybox_unstripped
@cp toybox_unstripped toybox_old
rm -rf toybox toybox_unstripped generated/config.h generated/Config.in \
generated/newtoys.h generated/globals.h instlist testdir \
- generated/Config.probed
+ generated/Config.probed generated/help.h generated/build_files
distclean: clean
- rm -f toybox_old .config* generated/help.h
+ rm -f toybox_old .config* generated/genconf
test: tests
diff --git a/scripts/config2help.py b/scripts/config2help.py
deleted file mode 100755
index 2573d08..0000000
--- a/scripts/config2help.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/python
-
-import os,sys
-
- if str[0]=='"': str = str[1:str.rfind('"')]
- return str
-
- return str.strip().replace("\\","\\\\").replace('"','\\"')
-
-helplen = morelines = 0
-out = sys.stdout
-
- global helplen, morelines
- #sys.stderr.write("Reading %s\n" % filename)
- lines = open(filename).read().split("\n")
- sys.stderr.write("File %s missing\n" % filename)
- return
- config = None
- description = None
- i = i.expandtabs()
- if morelines: out.write('\\n')
- morelines = 1
- out.write(escapequotes(i))
- continue
- helplen = morelines = 0
- out.write('"\n')
-
- words = i.strip().split(None,1)
- if not len(words): continue
-
- config = words[1]
- description = ""
- if len(words)>1: description = zapquotes(words[1])
- description = htmlescape(zapquotes(words[1]))
- out.write('#define help_%s "' % config.lower())
- helplen = len(i[:i.find(words[0])].expandtabs())
- elif words[0] == "source": readfile(zapquotes(words[1]))
- elif words[0] in ("default","depends", "select", "if", "endif", "#", "comment", "menu", "endmenu"): pass
-
-readfile(sys.argv[1])
-if helplen: out.write('"\n')
diff --git a/scripts/config2help.sh b/scripts/config2help.sh
deleted file mode 100755
index 63d6c6f..0000000
--- a/scripts/config2help.sh
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/bin/bash -e
-
-function firstmajor() {
- declare -i n j=$1
- test $j -gt 0 || return -1
- if [ $j -gt $1 ]; then
- echo $j
- return 0
- fi
- done
- return 1
-}; export -f firstmajor
-
-function print_h() {
- declare -i i c=$2 s=$3 e=$4
- local str="$(echo "$1" | head -n$c | tail -n1 | sed -e "s,config[\t ]*,,")"
- echo -n "#define help_"$str" \"" | tr [A-Z] [a-z]
- echo -n "$1\\n" | head -n$e | tail -n$[e-s+1] | sed -e "s,\$,\r," | tr \\n\\r n\\
- echo \"
-}; export -f print_h
-
-file="$1"
-if test "$0" != "bash"; then
- if test -r "$file"; then
-# echo "$file..." >&2
- filetxt="$(sed -e "s,^[\t ]*,," -e "s,\([\"\\\\]\),\\\\\\1,g" "$file")"
- helps=$(echo "$filetxt" | egrep -ne "^help *" -e "^---help--- *" | cut -d\: -f1)
- configs=$(echo "$filetxt" | egrep -ne "^config *" | cut -d\: -f1)
- endmenus=$(echo "$filetxt" | egrep -ne "^endmenu *" | cut -d\: -f1)
- let last=$(echo "$filetxt" | wc -l)+2
-
- declare -i i c s e
- for i in $configs; do
-# echo -n "c:$i" >&2
- s=$(firstmajor $i $helps)
- test $s -gt 0
- e=$(firstmajor $s $configs || firstmajor $s $endmenus $last)
- let s++ e-=2
- test $e -ge $s
-# echo " s:$s e:$e" >&2
- print_h "$filetxt" $i $s $e
- done
- for fle in $(cat "$file" | egrep -e "^[ \t]*source " | sed -e "s,^[ \t]*source *\(.*\),\\1,"); do
- $0 $fle
- done
- else
- echo
- echo "USAGE EXAMPLE: $(basename $0) Config.in > generated/help.h"
- echo
- false
- fi | sed -e "s,\\\\n\\\\n\"$,\\\\n\","
-fi
-
diff --git a/scripts/genconf.c b/scripts/genconf.c
new file mode 100644
index 0000000..8346dee
--- /dev/null
+++ b/scripts/genconf.c
@@ -0,0 +1,309 @@
+/* vi: set ts=4 :*/
+/*
+ * Generate toybox generated/{Config.in,config.h,globals.h,newtoys.h,help.h}
+ *
+ * Copyright 2012 Georgi Chorbadzhiyski <georgi at unixsol.org>
+ */
+__attribute__ ((format(printf, 2, 3)))
+void strbuf_printf(struct strbuf *s, const char *fmt, ...) {
This is a purely local function, the __attribute__ tag is overkill.
Post by Georgi Chorbadzhiyski
+ char line[4096];
+
+ va_list args;
+ va_start(args, fmt);
+ vsnprintf(line, sizeof(line) - 1, fmt, args);
+ va_end(args);
+
+ line[sizeof(line) - 1] = '\0';
+ strbuf_add(s, line);
+}
+
+void strbuf_add_escaped(struct strbuf *s, char *text) {
+ int len = strlen(text), i;
+ if (len == 0)
+ strbuf_add(s, "\\n");
+ for (i = 0; i < len; i++) {
+ char c[2] = { text[i], 0 };
+ switch (text[i]) {
+ case '\n': strbuf_add(s, "\\n"); break;
+ case '\t': strbuf_add(s, " "); break;
+ case '\\': strbuf_add(s, "\\\\"); break;
+ case '"' : strbuf_add(s, "\\\""); break;
+ default : strbuf_add(s, c); break;
+ }
+ }
+}
+
+void strbuf_dump(struct strbuf *s, char *filename, char *mode) {
+ FILE *fh;
+ if (!s->data) {
+ unlink(filename);
+ return;
+ }
+ fh = xfopen(filename, mode);
+ fwrite(s->data, s->len, 1, fh);
+ fclose(fh);
+ free(s->data);
+}
+
+void parse_help(char *filename) {
+ int i, nitems = 0;
+ char *config_name = NULL;
+
+ f = xfopen(filename, "r");
+ while (getline(&line, &linelen, f) > 0) {
+ if (line[0] == '#')
+ continue;
+ if (strncmp(line, "config ", 7) == 0) {
+ in_config = 1;
+ in_help = 0;
+ free(config_name);
+ config_name = strdup(line + 7);
+ int config_len = strlen(config_name);
+ for (i = 0; i < config_len; i++) {
+ config_name[i] = tolower((unsigned char)config_name[i]);
+ if (config_name[i] == '\n')
+ config_name[i] = '\0';
+ }
+ // Close already started help line
+ if (nitems)
+ strbuf_add(&help_h, "\"\n");
+ continue;
+ }
+ if (strncmp(line, "endmenu", 7) == 0) {
+ in_config = 0;
+ in_help = 0;
+ }
+ if (!in_config)
+ continue;
+
+ if (!in_help) {
+ if (strncmp(line, "\thelp", 5) == 0 || strncmp(line, " help", 8) == 0) {
+ in_help = 1;
+ nitems++;
+ strbuf_printf(&help_h, "#define help_%s \"", config_name);
+ }
+ continue;
+ } else {
+ char *tmpline = line;
+ while (isspace(*tmpline))
+ tmpline++;
+ strbuf_add_escaped(&help_h, tmpline);
+ continue;
+ }
+ }
+ strbuf_add(&help_h, "\"\n");
I was trying to detect sub-options, and also merge the "usage:" lines.
(This turn out to be a bit fiddly, you can use the "depends" lines which
can have "depends on ONE && TWO" so there's some parsing to do, or you
can depend on CONFIG_WALRUS and CONFIG_WALRUS_SUBOPT starting with the
same prefix, which gets fiddly when CONFIG_WALRUS has multiple options
and how do you tell that CONFIG_PIVOT_ROOT isn't a subopt for CONFIG_PIVOT?)
Post by Georgi Chorbadzhiyski
+void generate_headers(int argc, char **argv) {
+ // Parse toys
+ strbuf_add(&newtoys_h, "NEWTOY(toybox, NULL, 0)\n");
+ strbuf_add(&globals_union, "extern union global_union {\n");
It matters that these be in alphabetical order (for the binary search),
and we can't depend on the filesystem doing it for us. Where are you
sorting this?

I'll have to look at this more closely in the morning...

Rob
Continue reading on narkive:
Loading...