tools/mklibs: update to 0.1.44 and convert to Python 3

Since the switch to Python 3 build fails if CONFIG_USE_MKLIBS is set
("Strip unnecessary functions from libraries" in menuconfig) as
mklibs hasn't been converted to run on Python 3.

 * update to most recent upstream version which brings some
   reproducibility fixes
 * converted to Python 3 using 2to3
 * fixed mixed tab/spaces indentation
 * fixed use of string.* functions
 * some more minor fixes to make Python 3 happy

Fixes commit 19938c8de7 ("build: switch to Python 3")
Signed-off-by: Daniel Golle <daniel@makrotopia.org>

tools/mklibs/Makefile

@@ -7,17 +7,18 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=mklibs
-PKG_VERSION:=0.1.35
+PKG_VERSION:=0.1.44
 
-PKG_SOURCE:=$(PKG_NAME)_$(PKG_VERSION).tar.gz
+PKG_SOURCE:=$(PKG_NAME)_$(PKG_VERSION).tar.xz
 PKG_SOURCE_URL:=http://ftp.de.debian.org/debian/pool/main/m/mklibs/
-PKG_HASH:=ccb1023dc1729c5a37ca6c3eca8e4bac3491116763c8820dfce8eea4845c8567
+PKG_HASH:=3af0b6bd35e5b6fc58d8b68827fbae2ff6b7e20dd2b238ccb9b49d84722066c2
 
 HOST_FIXUP:=autoreconf
 
 include $(INCLUDE_DIR)/host-build.mk
 
 HOST_CFLAGS += -I$(CURDIR)/include
+HOST_CPPFLAGS += -std=gnu++98
 
 define Host/Install
        $(INSTALL_BIN) \

tools/mklibs/patches/002-disable_symbol_checks.patch

@@ -1,6 +1,6 @@
 --- a/src/mklibs
 +++ b/src/mklibs
-@@ -524,7 +524,7 @@ while 1:
+@@ -533,7 +533,7 @@ while 1:
          # No progress in last pass. Verify all remaining symbols are weak.
          for name in unresolved:
              if not needed_symbols[name].weak:
@@ -9,7 +9,7 @@
          break
  
      previous_pass_unresolved = unresolved
-@@ -559,7 +559,7 @@ while 1:
+@@ -568,7 +568,7 @@ while 1:
      for name in needed_symbols:
          if not name in symbol_provider:
              if not needed_symbols[name].weak:

tools/mklibs/patches/003-no_copy.patch

@@ -1,6 +1,6 @@
 --- a/src/mklibs
 +++ b/src/mklibs
-@@ -463,7 +463,7 @@ while 1:
+@@ -472,7 +472,7 @@ while 1:
      passnr = passnr + 1
      # Gather all already reduced libraries and treat them as objects as well
      small_libs = []
@@ -9,7 +9,7 @@
          obj = dest_path + "/" + lib
          small_libs.append(obj)
          inode = os.stat(obj)[ST_INO]
-@@ -579,12 +579,7 @@ while 1:
+@@ -588,12 +588,7 @@ while 1:
          if not so_file:
              sys.exit("File not found:" + library)
          pic_file = find_pic(library)
@@ -23,7 +23,7 @@
              # we have a pic file, recompile
              debug(DEBUG_SPAM, "extracting from:", pic_file, "so_file:", so_file)
              soname = extract_soname(so_file)
-@@ -627,22 +622,14 @@ while 1:
+@@ -636,22 +631,14 @@ while 1:
              cmd.append(library_depends_gcc_libnames(so_file))
              command(target + "gcc", *cmd)
  
@@ -47,4 +47,4 @@
 +    os.rename(dest_path + "/" + lib + "-so", dest_path + "/" + lib)
  
  # Canonicalize library names.
- for lib in regexpfilter(os.listdir(dest_path), "(.*so[.\d]*)$"):
+ for lib in sorted(regexpfilter(os.listdir(dest_path), "(.*so[.\d]*)$")):

tools/mklibs/patches/004-libpthread_link.patch

@@ -17,7 +17,7 @@
              if match.group('ld'):
                  ret.append(find_lib(match.group(0)))
              elif match.group('lib'):
-@@ -619,7 +619,7 @@ while 1:
+@@ -628,7 +628,7 @@ while 1:
              cmd.extend(extra_flags)
              cmd.append("-lgcc")
              cmd.extend(["-L%s" % a for a in [dest_path] + [sysroot + b for b in lib_path if sysroot == "" or b not in ("/" + libdir + "/", "/usr/" + libdir + "/")]])

tools/mklibs/patches/005-duplicate_syms.patch

@@ -1,6 +1,6 @@
 --- a/src/mklibs
 +++ b/src/mklibs
-@@ -531,7 +531,6 @@ while 1:
+@@ -540,7 +540,6 @@ while 1:
  
      library_symbols = {}
      library_symbols_used = {}
@@ -8,7 +8,7 @@
  
      # WORKAROUND: Always add libgcc on old-abi arm
      header = elf_header(find_lib(libraries.copy().pop()))
-@@ -549,20 +548,13 @@ while 1:
+@@ -558,20 +557,13 @@ while 1:
          library_symbols_used[library] = set()
          for symbol in symbols:
              for name in symbol.base_names():
@@ -32,4 +32,4 @@
 +                library_symbols_used[lib].add(library_symbols[lib][name])
  
      # reduce libraries
-     for library in libraries:
+     for library in sorted(libraries):

tools/mklibs/patches/006-uclibc_init.patch

@@ -1,6 +1,6 @@
 --- a/src/mklibs
 +++ b/src/mklibs
-@@ -595,6 +595,11 @@ while 1:
+@@ -604,6 +604,11 @@ while 1:
                  extra_post_obj.append(sysroot + libc_extras_dir + "/sofini.o")
                  symbols.add(ProvidedSymbol('__dso_handle', None, None, True))
  

tools/mklibs/patches/007-gc_sections.patch

@@ -1,6 +1,6 @@
 --- a/src/mklibs
 +++ b/src/mklibs
-@@ -607,7 +607,7 @@ while 1:
+@@ -616,7 +616,7 @@ while 1:
              # compile in only used symbols
              cmd = []
              cmd.extend(gcc_options)

tools/mklibs/patches/008-uclibc_libgcc_link.patch

@@ -14,7 +14,7 @@
      return ' '.join(ret)
  
  class Symbol(object):
-@@ -584,6 +581,7 @@ while 1:
+@@ -593,6 +590,7 @@ while 1:
              extra_flags = []
              extra_pre_obj = []
              extra_post_obj = []
@@ -22,7 +22,7 @@
  
              symbols.update(library_symbols_used[library])
  
-@@ -614,9 +612,10 @@ while 1:
+@@ -623,9 +621,10 @@ while 1:
              cmd.append(pic_file)
              cmd.extend(extra_post_obj)
              cmd.extend(extra_flags)

tools/mklibs/patches/009-uclibc_libpthread_symbols.patch

@@ -1,6 +1,6 @@
 --- a/src/mklibs
 +++ b/src/mklibs
-@@ -162,9 +162,10 @@ def undefined_symbols(obj):
+@@ -173,9 +173,10 @@ def undefined_symbols(obj):
      return result
  
  class ProvidedSymbol(Symbol):
@@ -12,7 +12,7 @@
  
      def base_names(self):
          ret = []
-@@ -205,11 +206,15 @@ def provided_symbols(obj):
+@@ -216,11 +217,15 @@ def provided_symbols(obj):
          if version_string.lower() not in ('base', 'none'):
              version = version_string
  
@@ -29,7 +29,7 @@
  
      return result
      
-@@ -500,6 +505,9 @@ while 1:
+@@ -509,6 +514,9 @@ while 1:
              debug(DEBUG_SPAM, "present_symbols adding %s" % symbol)
              names = symbol.base_names()
              for name in names:
@@ -39,7 +39,7 @@
                  present_symbols[name] = symbol
  
      # are we finished?
-@@ -591,12 +599,16 @@ while 1:
+@@ -600,12 +608,16 @@ while 1:
                  # may segfault in ptmalloc_init due to undefined weak reference
                  extra_pre_obj.append(sysroot + libc_extras_dir + "/soinit.o")
                  extra_post_obj.append(sysroot + libc_extras_dir + "/sofini.o")

tools/mklibs/patches/011-remove_multiarch.patch

@@ -1,6 +1,6 @@
 --- a/src/mklibs
 +++ b/src/mklibs
-@@ -263,6 +263,7 @@ def extract_soname(so_file):
+@@ -274,6 +274,7 @@ def extract_soname(so_file):
      return ""
  
  def multiarch(paths):

tools/mklibs/patches/100-apply-2to3.patch

@@ -0,0 +1,327 @@
+--- a/src/mklibs
++++ b/src/mklibs
+@@ -57,18 +57,18 @@ debuglevel = DEBUG_NORMAL
+ 
+ def debug(level, *msg):
+     if debuglevel >= level:
+-        print string.join(msg)
++        print(' '.join(msg))
+ 
+ # return a list of lines of output of the command
+ def command(command, *args):
+-    debug(DEBUG_SPAM, "calling", command, string.join(args))
++    debug(DEBUG_SPAM, "calling", command, ' '.join(args))
+     pipe = os.popen(command + ' ' + ' '.join(args), 'r')
+     output = pipe.read().strip()
+     status = pipe.close() 
+     if status is not None and os.WEXITSTATUS(status) != 0:
+-        print "Command failed with status", os.WEXITSTATUS(status),  ":", \
+-               command, string.join(args)
+-	print "With output:", output
++        print("Command failed with status", os.WEXITSTATUS(status),  ":", \
++               command, ' '.join(args))
++        print("With output:", output)
+         sys.exit(1)
+     return [i for i in output.split('\n') if i]
+ 
+@@ -204,7 +204,7 @@ class ProvidedSymbol(Symbol):
+ # Return a set of symbols provided by a library
+ def provided_symbols(obj):
+     if not os.access(obj, os.F_OK):
+-        raise Exception("Cannot find lib" + obj)
++        raise Exception("Cannot find lib " + obj)
+     library = extract_soname(obj)
+ 
+     output = command("mklibs-readelf", "--print-symbols-provided", obj)
+@@ -297,27 +297,27 @@ def usage(was_err):
+         outfd = sys.stderr
+     else:
+         outfd = sys.stdout
+-    print >> outfd, "Usage: mklibs [OPTION]... -d DEST FILE ..."
+-    print >> outfd, "Make a set of minimal libraries for FILE(s) in DEST."
+-    print >> outfd, "" 
+-    print >> outfd, "  -d, --dest-dir DIRECTORY     create libraries in DIRECTORY"
+-    print >> outfd, "  -D, --no-default-lib         omit default libpath (", ':'.join(default_lib_path), ")"
+-    print >> outfd, "  -L DIRECTORY[:DIRECTORY]...  add DIRECTORY(s) to the library search path"
+-    print >> outfd, "  -l LIBRARY                   add LIBRARY always"
+-    print >> outfd, "      --ldlib LDLIB            use LDLIB for the dynamic linker"
+-    print >> outfd, "      --libc-extras-dir DIRECTORY  look for libc extra files in DIRECTORY"
+-    print >> outfd, "      --target TARGET          prepend TARGET- to the gcc and binutils calls"
+-    print >> outfd, "      --root ROOT              search in ROOT for library rpaths"
+-    print >> outfd, "      --sysroot ROOT           prepend ROOT to all paths for libraries"
+-    print >> outfd, "      --gcc-options OPTIONS    pass OPTIONS to gcc"
+-    print >> outfd, "      --libdir DIR             use DIR (e.g. lib64) in place of lib in default paths"
+-    print >> outfd, "  -v, --verbose                explain what is being done"
+-    print >> outfd, "  -h, --help                   display this help and exit"
++    print("Usage: mklibs [OPTION]... -d DEST FILE ...", file=outfd)
++    print("Make a set of minimal libraries for FILE(s) in DEST.", file=outfd)
++    print("", file=outfd) 
++    print("  -d, --dest-dir DIRECTORY     create libraries in DIRECTORY", file=outfd)
++    print("  -D, --no-default-lib         omit default libpath (", ':'.join(default_lib_path), ")", file=outfd)
++    print("  -L DIRECTORY[:DIRECTORY]...  add DIRECTORY(s) to the library search path", file=outfd)
++    print("  -l LIBRARY                   add LIBRARY always", file=outfd)
++    print("      --ldlib LDLIB            use LDLIB for the dynamic linker", file=outfd)
++    print("      --libc-extras-dir DIRECTORY  look for libc extra files in DIRECTORY", file=outfd)
++    print("      --target TARGET          prepend TARGET- to the gcc and binutils calls", file=outfd)
++    print("      --root ROOT              search in ROOT for library rpaths", file=outfd)
++    print("      --sysroot ROOT           prepend ROOT to all paths for libraries", file=outfd)
++    print("      --gcc-options OPTIONS    pass OPTIONS to gcc", file=outfd)
++    print("      --libdir DIR             use DIR (e.g. lib64) in place of lib in default paths", file=outfd)
++    print("  -v, --verbose                explain what is being done", file=outfd)
++    print("  -h, --help                   display this help and exit", file=outfd)
+     sys.exit(was_err)
+ 
+ def version(vers):
+-    print "mklibs: version ",vers
+-    print ""
++    print("mklibs: version ",vers)
++    print("")
+ 
+ #################### main ####################
+ ## Usage: ./mklibs.py [OPTION]... -d DEST FILE ...
+@@ -368,8 +368,8 @@ script_pattern = re.compile("^#!\s*/")
+ 
+ try:
+     optlist, proglist = getopt.getopt(sys.argv[1:], opts, longopts)
+-except getopt.GetoptError, msg:
+-    print >> sys.stderr, msg
++except getopt.GetoptError as msg:
++    print(msg, file=sys.stderr)
+     usage(1)
+ 
+ for opt, arg in optlist:
+@@ -377,7 +377,7 @@ for opt, arg in optlist:
+         if debuglevel < DEBUG_SPAM:
+             debuglevel = debuglevel + 1
+     elif opt == "-L":
+-        lib_path.extend(string.split(arg, ":"))
++        lib_path.extend(arg.split(":"))
+     elif opt in ("-d", "--dest-dir"):
+         dest_path = arg
+     elif opt in ("-D", "--no-default-lib"):
+@@ -396,17 +396,17 @@ for opt, arg in optlist:
+     elif opt in ("-l",):
+         force_libs.append(arg)
+     elif opt == "--gcc-options":
+-        gcc_options.extend(string.split(arg, " "))
++        gcc_options.extend(arg.split(" "))
+     elif opt == "--libdir":
+         libdir = arg
+     elif opt in ("--help", "-h"):
+-	usage(0)
++        usage(0)
+         sys.exit(0)
+     elif opt in ("--version", "-V"):
+         version(vers)
+         sys.exit(0)
+     else:
+-        print "WARNING: unknown option: " + opt + "\targ: " + arg
++        print("WARNING: unknown option: " + opt + "\targ: " + arg)
+ 
+ if include_default_lib_path == "yes":
+     lib_path.extend([a.replace("/lib/", "/" + libdir + "/") for a in default_lib_path])
+@@ -424,22 +424,22 @@ if ldlib == "LDLIB":
+ objects = {}  # map from inode to filename
+ for prog in proglist:
+     inode = os.stat(prog)[ST_INO]
+-    if objects.has_key(inode):
++    if inode in objects:
+         debug(DEBUG_SPAM, prog, "is a hardlink to", objects[inode])
+     elif so_pattern.match(prog):
+         debug(DEBUG_SPAM, prog, "is a library")
+-    elif script_pattern.match(open(prog).read(256)):
++    elif script_pattern.match(open(prog, 'r', encoding='iso-8859-1').read(256)):
+         debug(DEBUG_SPAM, prog, "is a script")
+     else:
+         objects[inode] = prog
+ 
+ if not ldlib:
+-    for obj in objects.values():
++    for obj in list(objects.values()):
+         output = command("mklibs-readelf", "--print-interp", obj)
+         if output:
+             ldlib = output.pop()
+-	if ldlib:
+-	    break
++        if ldlib:
++            break
+ 
+ if not ldlib:
+     sys.exit("E: Dynamic linker not found, aborting.")
+@@ -454,10 +454,10 @@ for obj in sorted(objects.values()):
+             for rpath_elem in rpath_val:
+                 if not rpath_elem in lib_rpath:
+                     if debuglevel >= DEBUG_VERBOSE:
+-                        print "Adding rpath " + rpath_elem + " for " + obj
++                        print("Adding rpath " + rpath_elem + " for " + obj)
+                     lib_rpath.append(rpath_elem)
+         else:
+-            print "warning: " + obj + " may need rpath, but --root not specified"
++            print("warning: " + obj + " may need rpath, but --root not specified")
+ 
+ lib_path.extend(lib_rpath)
+ 
+@@ -465,12 +465,12 @@ passnr = 1
+ available_libs = []
+ previous_pass_unresolved = set()
+ while 1:
+-    debug(DEBUG_NORMAL, "I: library reduction pass", `passnr`)
++    debug(DEBUG_NORMAL, "I: library reduction pass", repr(passnr))
+     if debuglevel >= DEBUG_VERBOSE:
+-        print "Objects:",
+-        for obj in sorted([x[string.rfind(x, '/') + 1:] for x in objects.values()]):
+-            print obj,
+-        print
++        print("Objects:", end=' ')
++        for obj in sorted([x[x.rfind('/') + 1:] for x in list(objects.values())]):
++            print(obj, end=' ')
++        print()
+ 
+     passnr = passnr + 1
+     # Gather all already reduced libraries and treat them as objects as well
+@@ -479,7 +479,7 @@ while 1:
+         obj = dest_path + "/" + lib
+         small_libs.append(obj)
+         inode = os.stat(obj)[ST_INO]
+-        if objects.has_key(inode):
++        if inode in objects:
+             debug(DEBUG_SPAM, obj, "is hardlink to", objects[inode])
+         else:
+             objects[inode] = obj
+@@ -509,7 +509,7 @@ while 1:
+     present_symbols = {}
+     checked_libs = small_libs
+     checked_libs.extend(available_libs)
+-    checked_libs.append(ldlib)
++    checked_libs.append(sysroot + "/" + ldlib)
+     for lib in checked_libs:
+         for symbol in provided_symbols(lib):
+             debug(DEBUG_SPAM, "present_symbols adding %s" % symbol)
+@@ -529,8 +529,8 @@ while 1:
+             unresolved.add(name)
+             num_unresolved = num_unresolved + 1
+ 
+-    debug (DEBUG_NORMAL, `len(needed_symbols)`, "symbols,",
+-           `num_unresolved`, "unresolved")
++    debug (DEBUG_NORMAL, repr(len(needed_symbols)), "symbols,",
++           repr(num_unresolved), "unresolved")
+ 
+     if num_unresolved == 0:
+         break
+@@ -539,7 +539,7 @@ while 1:
+         # No progress in last pass. Verify all remaining symbols are weak.
+         for name in unresolved:
+             if not needed_symbols[name].weak:
+-                print "WARNING: Unresolvable symbol %s" % name
++                print("WARNING: Unresolvable symbol %s" % name)
+         break
+ 
+     previous_pass_unresolved = unresolved
+@@ -641,9 +641,9 @@ while 1:
+             command(target + "gcc", *cmd)
+ 
+             ## DEBUG
+-            debug(DEBUG_VERBOSE, so_file, "\t", `os.stat(so_file)[ST_SIZE]`)
++            debug(DEBUG_VERBOSE, so_file, "\t", repr(os.stat(so_file)[ST_SIZE]))
+             debug(DEBUG_VERBOSE, dest_path + "/" + so_file_name + "-so", "\t",
+-                  `os.stat(dest_path + "/" + so_file_name + "-so")[ST_SIZE]`)
++                  repr(os.stat(dest_path + "/" + so_file_name + "-so")[ST_SIZE]))
+ 
+ # Finalising libs and cleaning up
+ for lib in regexpfilter(os.listdir(dest_path), "(.*)-so$"):
+@@ -680,4 +680,4 @@ if not os.access(dest_path + "/" + ld_fu
+     command(target + "objcopy", "--strip-unneeded -R .note -R .comment",
+             ld_file, dest_path + "/" + ld_full_path)
+ 
+-os.chmod(dest_path + "/" + ld_full_path, 0755)
++os.chmod(dest_path + "/" + ld_full_path, 0o755)
+--- a/src/mklibs-copy
++++ b/src/mklibs-copy
+@@ -51,9 +51,9 @@ def command(command, *args):
+     output = pipe.read().strip()
+     status = pipe.close()
+     if status is not None and os.WEXITSTATUS(status) != 0:
+-        print "Command failed with status", os.WEXITSTATUS(status),  ":", \
+-               command, ' '.join(args)
+-	print "With output:", output
++        print("Command failed with status", os.WEXITSTATUS(status),  ":", \
++               command, ' '.join(args))
++        print("With output:", output)
+         sys.exit(1)
+     return output.split('\n')
+ 
+@@ -134,8 +134,8 @@ def multiarch(paths):
+         return paths
+ 
+ def version(vers):
+-    print "mklibs: version ",vers
+-    print ""
++    print("mklibs: version ",vers)
++    print("")
+ 
+ # Clean the environment
+ vers="0.12"
+@@ -159,7 +159,7 @@ if include_default_lib_path:
+ objects = {}  # map from inode to filename
+ for prog in proglist:
+     inode = os.stat(prog)[ST_INO]
+-    if objects.has_key(inode):
++    if inode in objects:
+         logger.debug("%s is a hardlink to %s", prog, objects[inode])
+     elif so_pattern.match(prog):
+         logger.debug("%s is a library", prog)
+@@ -169,12 +169,12 @@ for prog in proglist:
+         logger.debug("%s is no ELF", prog)
+ 
+ if not ldlib:
+-    for obj in objects.values():
++    for obj in list(objects.values()):
+         output = command("mklibs-readelf", "-i", obj)
+-	for x in output:
++        for x in output:
+             ldlib = x
+-	if ldlib:
+-	    break
++        if ldlib:
++            break
+ 
+ if not ldlib:
+     sys.exit("E: Dynamic linker not found, aborting.")
+@@ -182,7 +182,7 @@ if not ldlib:
+ logger.info('Using %s as dynamic linker', ldlib)
+ 
+ # Check for rpaths
+-for obj in objects.values():
++for obj in list(objects.values()):
+     rpath_val = rpath(obj)
+     if rpath_val:
+         if root:
+@@ -208,18 +208,18 @@ while 1:
+         obj = dest_path + "/" + lib
+         small_libs.append(obj)
+         inode = os.stat(obj)[ST_INO]
+-        if objects.has_key(inode):
++        if inode in objects:
+             logger.debug("%s is hardlink to %s", obj, objects[inode])
+         else:
+             objects[inode] = obj
+ 
+-    for obj in objects.values():
++    for obj in list(objects.values()):
+         small_libs.append(obj)
+ 
+-    logger.verbose('Objects: %r', ' '.join([i[i.rfind('/') + 1:] for i in objects.itervalues()]))
++    logger.verbose('Objects: %r', ' '.join([i[i.rfind('/') + 1:] for i in objects.values()]))
+ 
+     libraries = set()
+-    for obj in objects.values():
++    for obj in list(objects.values()):
+         libraries.update(library_depends(obj))
+ 
+     if libraries == previous_pass_libraries:
+@@ -272,4 +272,4 @@ if not os.access(dest_path + "/" + ld_fu
+     command(target + "objcopy", "--strip-unneeded -R .note -R .comment",
+             ld_file, dest_path + "/" + ld_full_path)
+ 
+-os.chmod(dest_path + "/" + ld_full_path, 0755)
++os.chmod(dest_path + "/" + ld_full_path, 0o755)