diff options
| -rw-r--r-- | src/grp-system/libcore/src/load-fragment-gperf.gperf.m4 | 7 | ||||
| -rw-r--r-- | src/libsystemd/src/sd-journal/audit-type.c | 2 | ||||
| -rw-r--r-- | tools/.gitignore | 1 | ||||
| -rwxr-xr-x | tools/notsd-fixup | 57 | ||||
| -rwxr-xr-x | tools/notsd-fixup--includes | 540 | ||||
| -rwxr-xr-x | tools/notsd-fixup--makefiles | 30 | ||||
| -rwxr-xr-x | tools/notsd-move | 14 |
7 files changed, 364 insertions, 287 deletions
diff --git a/src/grp-system/libcore/src/load-fragment-gperf.gperf.m4 b/src/grp-system/libcore/src/load-fragment-gperf.gperf.m4 index 6a5c16a000..4658fe71b8 100644 --- a/src/grp-system/libcore/src/load-fragment-gperf.gperf.m4 +++ b/src/grp-system/libcore/src/load-fragment-gperf.gperf.m4 @@ -1,8 +1,9 @@ %{ #include <stddef.h> -#include "conf-parser.h" -#include "load-fragment.h" -#include "missing.h" + +#include "core/load-fragment.h" +#include "systemd-basic/missing.h" +#include "systemd-shared/conf-parser.h" %} struct ConfigPerfItem; %null_strings diff --git a/src/libsystemd/src/sd-journal/audit-type.c b/src/libsystemd/src/sd-journal/audit-type.c index b4fe13e8da..1644e3f1b1 100644 --- a/src/libsystemd/src/sd-journal/audit-type.c +++ b/src/libsystemd/src/sd-journal/audit-type.c @@ -24,8 +24,8 @@ # include <libaudit.h> #endif -#include "audit_type-to-name.h" #include "systemd-basic/macro.h" #include "systemd-basic/missing.h" #include "audit-type.h" +#include "audit_type-to-name.h" diff --git a/tools/.gitignore b/tools/.gitignore new file mode 100644 index 0000000000..4bba404d19 --- /dev/null +++ b/tools/.gitignore @@ -0,0 +1 @@ +/notsd-fixup--includes.cache diff --git a/tools/notsd-fixup b/tools/notsd-fixup index 99ae0981a3..71b3bf4dfa 100755 --- a/tools/notsd-fixup +++ b/tools/notsd-fixup @@ -1,50 +1,25 @@ #!/usr/bin/env bash - -# The reason we do `find`/`while read`-loops instead of `find -exec` commands -# is that we want errors from the inner loop to bubble up. - -fixup_makefiles() ( - find "$@" -type f -name Makefile | while read -r filename; do - { - <"$filename" sed -r \ - -e "s|(/\.\.)*/config.mk|/$(realpath -ms --relative-to="$(dirname -- "$filename")" config.mk)|" \ - -e '/^nested\.subdirs/d' \ - -e '/^include \$\(topsrcdir\)\/build-aux\/Makefile\.tail\.mk$/d' - echo - find "$(dirname "$filename")" -mindepth 2 -maxdepth 2 -name Makefile -print0 | - xargs -r0 dirname -z -- | - xargs -r0 basename -a -z | - xargs -r0 printf 'nested.subdirs += %s\n' | sort - echo - echo 'include $(topsrcdir)/build-aux/Makefile.tail.mk' - } | cat -s | build-aux/write-ifchanged "$filename" - done -) - -fixup_includes() ( - dirs=($(find "$@" -type d -name include)) - if [[ ${#dirs[@]} -gt 0 ]]; then - find "${dirs[@]}" -type d | while read -r dir; do - printf '=> libdir %q\n' "$dir" - lib="${dir##*/}" - find "$dir" -type f | while read -r filename; do - printf ' => sed -ir %q %q\n' "s|$lib/||" "$filename" - sed -r "s|$lib/||" < "$filename" | build-aux/write-ifchanged "$filename" - done - done - fi - - find "$@" \( -name '*.h' -o -name '*.c' -o -name '*.gperf' \) -type f | while read -r filename; do - "$0"--includes "$filename" - done -) +# Copyright (C) 2015-2016 Luke Shumaker main() { set -e set -o pipefail export LC_COLLATE=C - fixup_makefiles "$@" - fixup_includes "$@" + + # We wrap the programs called by xargs with `sh` because xargs only exits early + # if the status is 255, but we want to exit early for all non-zero statuses. + # We use xargs instead of `find -exec` because `-exec` won't do much of + # anything useful with the exit status. + + # Makefiles + find "$@" -type f -name Makefile -print0 | + xargs -r0 sh -c "$0--makefiles \"\$@\" || exit 255" -- + + # C includes + rm -rf -- "$0"--includes.cache + find "$@" \( -name '*.h' -o -name '*.c' -o -name '*.gperf' -o -name '*.gperf.m4' \) -type f -print0 | + xargs -r0 sh -c "$0--includes \"\$@\" || exit 255" -- + rm -rf -- "$0"--includes.cache } main "$@" diff --git a/tools/notsd-fixup--includes b/tools/notsd-fixup--includes index 33218233d5..a636c78be6 100755 --- a/tools/notsd-fixup--includes +++ b/tools/notsd-fixup--includes @@ -1,235 +1,305 @@ -#!/usr/bin/env bash - -panic() { - >&2 echo panic - exit 2 -} - -out() { - printf '%q ' "$@" -} - -# system -# linux -# public -# protected -# private -classify() { - local path=$1 - if [[ "$path" = linux/* ]]; then - out linux "$path" - elif [[ -f "${current_file%/*}/${path}" ]]; then - out private "$path" - elif [[ "$path" != systemd/* ]] && - [[ "$path" != libudev.h ]] && - cpp -include "$path" <<<'' &>/dev/null; then - out system "$path" - else - case "$path" in - *-to-name.h|*-from-name.h) - base="${path##*/}" - base="${base%-to-name.h}" - base="${base%-from-name.h}" - case "$base" in - dns_type) d=src/grp-resolve/systemd-resolved;; - keyboard-keys) d=src/grp-udev/libudev-core;; - af|arphrd|cap|errno) d=src/libsystemd-basic/src;; - audit_type) d=src/libsystemd/src/sd-journal;; - *) - >&2 printf 'Unknown gperf base: %q\n' "$base" - >&2 printf 'Cannot figure out: %q\n' "$path" - exit 2 - ;; - esac - file="$d/${path##*/}" - if [[ "$current_file" = "$d"/* ]]; then - out private "${file##*/}" - elif [[ "$file" = */include/* ]]; then - out protected "${file##*/include/}" - else - out protected "${file##*/}" - fi - ;; - asm/sgidefs.h|dbus/dbus.h|efi.h|efilib.h|gio/gio.h|glib.h|libmount.h) - out system "$path" - ;; - util.h|*/util.h) - if [[ "$current_file" = */systemd-boot/* ]]; then - out private util.h - else - out protected systemd-basic/util.h - fi - ;; - *) - file=$(find src -type f -name "${path##*/}") - if [[ -f "$file" ]]; then - case "$file" in - */src/*) - if [[ "${current_file%/*}" = "${file%/*}" ]]; then - out private "${file##*/}" - else - out protected "${file##*/src/}" - fi - ;; - */libsystemd/include/*|*/libudev/include/*) - out public "${file##*/include/}" - ;; - */include/*) - out protected "${file##*/include/}" - ;; - */include-staging/*) - out protected "${file##*/include-staging/}" - ;; - *) - if [[ "${current_file%/*}" = "${file%/*}" ]]; then - out private "${file##*/}" - else - out protected "${file##*/}" - fi - ;; - esac - else - >&2 printf 'Cannot figure out: %q\n' "$path" - exit 2 - fi - ;; - esac - fi -} - -phase=phase0 -hook=: - -phase0() { - phase=phase0 - hook=: - local line="$1" - case "$line" in - '#include'*|'typedef '*';') - phase1 "$line" - ;; - *) - printf '%s\n' "$line" - ;; - esac -} - -phase1_init() { - phase1_tail= - system=() - linux=() - public=() - protected=() - typedef=(); typedef_last=true - private=() -} -phase1_init -phase1_flush() { - local b=: - if [[ ${#system[@]} -gt 0 ]]; then - printf '%s\n' "${system[@]}" | sort -u - b=echo - fi - if [[ ${#linux[@]} -gt 0 ]]; then - $b - printf '%s\n' "${linux[@]}" - b=echo - fi - if [[ ${#public[@]} -gt 0 ]]; then - $b - printf '%s\n' "${public[@]}" | sort -u - b=echo - fi - if [[ ${#protected[@]} -gt 0 ]]; then - $b - printf '%s\n' "${protected[@]}" | sort -u - b=echo - fi - if [[ ${#typedef[@]} -gt 0 ]] && ! $typedef_last; then - $b - printf '%s\n' "${typedef[@]}" | sort -u - b=echo - fi - if [[ ${#private[@]} -gt 0 ]]; then - $b - printf '%s\n' "${private[@]}" | sort -u - b=echo - fi - if [[ ${#typedef[@]} -gt 0 ]] && $typedef_last; then - $b - printf '%s\n' "${typedef[@]}" - fi - printf '%s' "$phase1_tail" - phase1_init -} -phase1() { - phase=phase1 - hook=phase1_flush - local line="$1" - case "$line" in - '') - phase1_tail+=$'\n' - ;; - '#include'*) - phase1_tail='' - local re='^#include [<"]([^">]*)[">](.*)' - if [[ "$line" =~ $re ]]; then - IFS=' ' - local buf - buf="$(classify "${BASH_REMATCH[1]}")" || panic - read -r class path <<<"$buf" - case "$class" in - system) - printf -v line '#include <%s>%s' "$path" "${BASH_REMATCH[2]}" - system+=("$line") - ;; - linux) - printf -v line '#include <%s>%s' "$path" "${BASH_REMATCH[2]}" - linux+=("$line") - ;; - public) - printf -v line '#include <%s>%s' "$path" "${BASH_REMATCH[2]}" - public+=("$line") - ;; - protected) - printf -v line '#include "%s"%s' "$path" "${BASH_REMATCH[2]}" - protected+=("$line") - ;; - private) - if [[ ${#typedef[@]} -gt 0 ]]; then - typedef_last=false - fi - printf -v line '#include "%s"%s' "$path" "${BASH_REMATCH[2]}" - private+=("$line") - ;; - esac - else - panic - fi - ;; - 'typedef '*';') - phase1_tail='' - typedef+=("$line") - ;; - *) - phase1_flush - phase0 "$line" - ;; - esac -} - -main() { - current_file="$1" - printf ' => %q %q\n' "$0" "$current_file" - set -o pipefail - { - IFS='' - while read -r line; do - "$phase" "$line" - IFS='' - done - "$hook" - } < "$current_file" | build-aux/write-ifchanged "$current_file" -} - -main "$@" +#!/usr/bin/env python3 + +# If you are thinking "this file looks gross!", it is. It +# started out as a set of Bash one-liners. Which got turned +# into a script. Which grew somewhat organically. Not huge, +# but given that it started as some one liners, that's not a +# very pretty several hunderd lines. Then got fairly litterally +# translated into this, for speed. So yes, it is gross. +# Rewrites welcome; just don't introduce any behavioral changes +# (easy since `tools/notsd-move` runs it on the entire repo and +# puts the results in git history). + +import atexit +import filecmp +import json +import os +import re +import shlex +import subprocess +import sys + +################################################################ +# Everything else in this program is just fluff and bookkeeping +# around around calling classify(). + +# Return a tuple of (class/group, path); which is a class that +# the header path belongs to, and a normalized path for it. +# +# There are a fixed number of classes that it may put a header +# in; in order of most-public to most-private: +# +# system +# linux +# public +# protected +# private +def classify(expensive, current_file, path): + if re.fullmatch('.*/include(-staging)?/.*/.*', current_file): + lib = os.path.basename(os.path.dirname(current_file)) + if path.startswith(lib+'/'): + path = re.sub('^'+lib+'/', path) + if path.startswith('linux/'): + return 'linux', path + elif expensive.exists(os.path.join(os.path.dirname(current_file), path)): + return 'private', path + elif not path.startswith('systemd/') and path != 'libudev.h' and expensive.cpp(path): + return 'system', path + else: + if path.endswith('-to-name.h') or path.endswith('-from-name.h'): + base = re.fullmatch('(.*)-(to|from)-name\.h', os.path.basename(path)).group(1) + d={ + 'dns_type' : 'src/grp-resolve/systemd-resolved', + 'keyboard-keys' : 'src/grp-udev/libudev-core', + 'af' : 'src/libsystemd-basic/src', + 'arphrd' : 'src/libsystemd-basic/src', + 'cap' : 'src/libsystemd-basic/src', + 'errno' : 'src/libsystemd-basic/src', + 'audit_type' : 'src/libsystemd/src/sd-journal', + } + file = os.path.join(d[base], os.path.basename(path)) + if current_file.startswith(d[base]): + return 'private', os.path.basename(file) + elif '/include/' in file: + return 'protected', re.sub('.*/include/', '', file) + else: + return 'protected', os.path.basename(file) + elif path in [ 'asm/sgidefs.h', 'dbus/dbus.h', 'efi.h', 'efilib.h', 'gio/gio.h', 'glib.h', 'libmount.h' ]: + return 'system', path + elif os.path.basename(path) == 'util.h': + if '/systemd-boot/' in current_file: + return 'private', 'util.h' + else: + return 'protected', 'systemd-basic/util.h' + else: + find = expensive.find(os.path.basename(path)) + if len(find) == 1: + file = find[0] + if '/src/' in file: + if os.path.dirname(current_file) == os.path.dirname(file): + return 'private', os.path.basename(file) + else: + return 'protected', re.sub('.*/src/', '', file) + elif ('/libsystemd/include/' in file) or ('/libudev/include/' in file): + return 'public', re.sub('.*/include/', '', file) + elif '/include/' in file: + return 'protected', re.sub('.*/include/', '', file) + elif '/include-staging/' in file: + return 'protected', re.sub('.*/include-staging/', '', file) + else: + if os.path.dirname(current_file) == os.path.dirname(file): + return 'private', os.path.basename(file) + else: + return 'protected', os.path.basename(file) + else: + sys.exit('Cannot figure out: {0}'.format(path)) + +################################################################ +# Cache expensive things + +class Cache: + def __init__(self, filename): + self.cache = { + 'find': None, + 'cpp': {} + } + self.dirty = True + + if os.path.isfile(filename): + with open(filename) as file: + self.cache = json.load(file) + self.dirty = False + + def save(self, filename): + if self.dirty: + with open(filename, 'w') as file: + json.dump(self.cache, file) + + def real_cpp(path): + # `cpp -include "$path" <<<'' &>/dev/null` + print(' -> cpp({0})'.format(path), file=sys.stderr) + with subprocess.Popen(['cpp', '-include', path], + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) as proc: + proc.stdin.close() + return proc.wait() == 0 + + def real_find(): + # This can probably be done with os.walk or something, + # but since it is only called once, it isn't a good + # place to start optimizing. + # + # `find src -name '*.h' \( -type l -printf 'l %p\n' -o -type f -printf 'f %p\n' \)` + print(' -> find()', file=sys.stderr) + ret = {} + with subprocess.Popen(['find', 'src', '-name', '*.h', '(', '-type', 'l', '-printf', 'l %p\n', '-o', '-type', 'f', '-printf', 'f %p\n', ')'], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, universal_newlines=True, + stderr=subprocess.DEVNULL) as proc: + for line in proc.stdout: + t, p = line.rstrip('\n').split(' ', 1) + ret[p]=t + return ret + + def cpp(self, path): + # `cpp -include "$path" <<<'' &>/dev/null` + if path not in self.cache['cpp']: + self.cache['cpp'][path] = Cache.real_cpp(path) + self.dirty = True + return self.cache['cpp'][path] + + def exists(self, path): + # `test -f "$path"` + if not self.cache['find']: + self.cache['find'] = Cache.real_find() + self.dirty = True + return path in self.cache['find'] + + def find(self, name): + # `find src -type f -name "$name"` + if not self.cache['find']: + self.cache['find'] = Cache.real_find() + self.dirty = True + return [p for p in self.cache['find'].keys() if self.cache['find'][p]=='f' and os.path.basename(p) == name] + +################################################################ +# Data structure for storing a chunk of `#include` lines. + +class IncludeSection: + def __init__(self): + self.trailing_nl = '' + self.system = [] + self.linux = [] + self.public = [] + self.protected = [] + self.typedef = [] + self.typedef_last = True + self.private = [] + def print(self, file=sys.stdout): + b='' + if len(self.system) > 0: + for line in sorted(set(self.system)): + print(line, file=file) + b='\n' + if len(self.linux) > 0: + print(b, end='', file=file) + for line in self.linux: + print(line, file=file) + b='\n' + if len(self.public) > 0: + print(b, end='', file=file) + for line in sorted(set(self.public)): + print(line, file=file) + b='\n' + if len(self.protected) > 0: + print(b, end='', file=file) + for line in sorted(set(self.protected)): + print(line, file=file) + b='\n' + if len(self.typedef) > 0 and not self.typedef_last: + print(b, end='', file=file) + for line in sorted(set(self.typedef)): + print(line, file=file) + b='\n' + if len(self.private) > 0: + print(b, end='', file=file) + for line in sorted(set(self.private)): + print(line, file=file) + b='\n' + if len(self.typedef) > 0 and self.typedef_last: + print(b, end='', file=file) + for line in self.typedef: + print(line, file=file) + print(self.trailing_nl, end='', file=file) + def add(self, group, path, extra): + if group == 'system': + self.system.append('#include <{0}>{1}'.format(path, extra)) + elif group == 'linux': + self.linux.append('#include <{0}>{1}'.format(path, extra)) + elif group == 'public': + self.public.append('#include <{0}>{1}'.format(path, extra)) + elif group == 'protected': + self.protected.append('#include "{0}"{1}'.format(path, extra)) + elif group == 'private': + if len(self.typedef) > 0: + self.typedef_last = False + self.private.append('#include "{0}"{1}'.format(path, extra)) + else: + sys.exit('panic: unrecognized line class: {0}'.format(group)) + +################################################################ +# The main program loop + +class Parser: + def __init__(self, cache, ifilename, ofilename): + self.cache = cache + self.ifilename = os.path.normpath(ifilename) + self.ofilename = ofilename + + self.includes = None + self.phase = self.phase0 + + def phase0(self, line, ofile): + self.phase = self.phase0 + + if re.fullmatch('#include.*|typedef .*;', line): + self.includes = IncludeSection() + self.phase1(line, ofile) + else: + print(line, file=ofile) + + def phase1(self, line, ofile): + self.phase = self.phase1 + + if line == '': + self.includes.trailing_nl += '\n' + elif line.startswith('#include'): + self.includes.trailing_nl = '' + match = re.fullmatch('^#include [<"]([^">]*)[">](.*)', line) + if match: + group, path = classify(self.cache, self.ifilename, match.group(1)) + self.includes.add(group, path, match.group(2)) + else: + sys.exit('panic: malformed #include line') + elif re.fullmatch('typedef .*;', line): + self.includes.trailing_nl = '' + self.includes.typedef.append(line) + else: + self.includes.print(file=ofile) + self.includes = None + self.phase0(line, ofile) + + def run(self): + print(' => {0} {1}'.format( + shlex.quote(__file__), + shlex.quote(self.ifilename), + ), file=sys.stderr) + with open(self.ofilename, 'w') as ofile: + with open(self.ifilename) as ifile: + for line in ifile: + self.phase(line.rstrip('\n'), ofile) + if self.includes: + self.includes.print(file=ofile) + +def main(argv): + cache = Cache(__file__+'.cache') + tmpfilename = '' + def cleanup(): + if tmpfilename != '': + try: + os.unlink(tmpfilename) + except FileNotFoundError: + pass + atexit.register(cleanup) + for filename in argv[1:]: + tmpfilename = os.path.join(os.path.dirname(filename), '.tmp.'+os.path.basename(filename)+'.tmp') + Parser(cache, filename, tmpfilename).run() + if not filecmp.cmp(filename, tmpfilename): + os.rename(tmpfilename, filename) + cleanup() + tmpfilename = '' + cache.save(__file__+'.cache') + +if __name__ == '__main__': + main(sys.argv) diff --git a/tools/notsd-fixup--makefiles b/tools/notsd-fixup--makefiles new file mode 100755 index 0000000000..f89e4b075b --- /dev/null +++ b/tools/notsd-fixup--makefiles @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +doit() { + local filename=$1 + { + <"$filename" sed -r \ + -e "s|(/\.\.)*/config.mk|/$(realpath -ms --relative-to="$(dirname -- "$filename")" config.mk)|" \ + -e '/^nested\.subdirs/d' \ + -e '/^include \$\(topsrcdir\)\/build-aux\/Makefile\.tail\.mk$/d' + echo + find "$(dirname "$filename")" -mindepth 2 -maxdepth 2 -name Makefile -print0 | + xargs -r0 dirname -z -- | + xargs -r0 basename -a -z | + xargs -r0 printf 'nested.subdirs += %s\n' | sort + echo + echo 'include $(topsrcdir)/build-aux/Makefile.tail.mk' + } | cat -s | build-aux/write-ifchanged "$filename" +} + +main() { + set -e + set -o pipefail + local filename + for filename in "$@"; do + >&2 printf ' => %q %q\n' "$0" "$filename" + doit "$filename" + done +} + +main "$@" diff --git a/tools/notsd-move b/tools/notsd-move index a2190bac64..23469ba407 100755 --- a/tools/notsd-move +++ b/tools/notsd-move @@ -709,6 +709,12 @@ move_files() ( mv -t src/grp-"$base" "$file" done rmdir system-preset + + ln -srT src/libsystemd/include/systemd/_sd-common.h src/libsystemd-network/include/systemd-network/_sd-common.h + ln -srT src/libsystemd/include/systemd/_sd-common.h src/grp-utils/systemd-path/_sd-common.h + ln -srT src/grp-login/systemd-logind/logind-acl.h src/grp-udev/libudev-core/logind-acl.h + ln -srT src/grp-login/systemd-logind/logind-acl.c src/grp-udev/libudev-core/logind-acl.c + ln -srT src/libsystemd/src/sd-login/sd-login.c src/grp-udev/libudev-core/sd-login.c ) breakup_makefile() ( @@ -756,12 +762,6 @@ breakup_makefile() ( src/libsystemd/src/sd-journal/Makefile \ src/grp-udev/libudev-core/Makefile - ln -srT src/libsystemd/include/systemd/_sd-common.h src/libsystemd-network/include/systemd-network/_sd-common.h - ln -srT src/libsystemd/include/systemd/_sd-common.h src/grp-utils/systemd-path/_sd-common.h - ln -srT src/grp-login/systemd-logind/logind-acl.h src/grp-udev/libudev-core/logind-acl.h - ln -srT src/grp-login/systemd-logind/logind-acl.c src/grp-udev/libudev-core/logind-acl.c - ln -srT src/libsystemd/src/sd-login/sd-login.c src/grp-udev/libudev-core/sd-login.c - ln -sT ../subdir.mk src/libsystemd/src/sd-network/Makefile ln -sT ../subdir.mk src/libsystemd/src/sd-bus/Makefile ln -sT ../subdir.mk src/libsystemd/src/sd-event/Makefile @@ -852,7 +852,7 @@ main() { git commit -m './tools/notsd-move' git merge -s ours notsystemd/postmove git checkout notsystemd/postmove - git merge tmp/postmove + git merge --no-edit tmp/postmove git branch -d tmp/postmove } |