Signing from scratch seems to currently be broken

[rib]

If I've cross-compiled a binary via clang + ld64 (without using XCode) then the code path for signing from scratch doesn't seem to be working currently.

The first issue I hit was that makesig.py doesn't have a line like:

log = logging.getLogger(__name__)

(so any log.xyz() calls result in an exception)

I guess it implies that no one else is trying to do this and maybe the code has bitrotted? :/

Then I git an issue that I guess was introduced with the python3 port maybe being more fussy about strings/binary data and currently have a hack like:

diff --git a/isign/makesig.py b/isign/makesig.py
index 736d3f3..3def60a 100644
--- a/isign/makesig.py
+++ b/isign/makesig.py
@@ -15,6 +15,7 @@ from . import macho
 from . import macho_cs
 from . import utils
 
+log = logging.getLogger(__name__)
 
 def make_arg(data_type, arg):
     if data_type.name == 'Data':
@@ -119,7 +120,7 @@ def make_basic_codesig(entitlements_file, drs, code_limit, hashes, signer, ident
     empty_hash = "\x00" * 20
 
     if not signer.is_adhoc():
-        teamID = signer._get_team_id() + '\x00'
+        teamID = signer._get_team_id() + '\x00'.encode('ascii')
         cd = construct.Container(cd_start=None,
                                  version=0x20200,
                                  flags=0,

I currently get this output running with --verbose

Signing with apple_cert: /home/bob/.local/lib/python3.6/site-packages/isign/apple_credentials/applecerts.pem
Signing with key: /home/bob/.isign/key.pem
Signing with certificate: /home/bob/.isign/certificate.pem
Signing with provisioning_profile: /home/bob/.isign/isign.mobileprovision
is_native: True
File <snip/>/Payload/Test.app matched as AppArchive
unarchiving to temp... <snip/>/Payload/Test.app -> /tmp/isign-e6q3qjfs
wrote Entitlements to /tmp/isign-e6q3qjfs/Entitlements.plist
working on /tmp/isign-e6q3qjfs/Test
found thin binary: cputype 16777228, cpusubtype 0
codesig len: 704
Existing LC_CODE_SIGNATURE missing entitlements
signing from scratch!
signing from scratch
codesig offset: 113504
new cL: 0x1bb60
new nCS: 28
ident: <snip/>.Test
codelimit: 113504
removing ua: /tmp/isign-e6q3qjfs
Traceback (most recent call last):
  File "/home/bob/.local/lib/python3.6/site-packages/construct/adapters.py", line 100, in _decode
    return self.decoding[obj]
KeyError: 16777216

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 444, in _parse
    obj.append(self.subcon._parse(stream, context))
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 672, in _parse
    subobj = sc._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 976, in _parse
    obj = self.subcon._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 1295, in _parse
    return self.bound._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 672, in _parse
    subobj = sc._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 1011, in _parse
    return self.subcon._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 853, in _parse
    obj = self.cases.get(key, self.default)._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 672, in _parse
    subobj = sc._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 1295, in _parse
    return self.bound._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 672, in _parse
    subobj = sc._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 853, in _parse
    obj = self.cases.get(key, self.default)._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 735, in _parse
    subobj = sc._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 1295, in _parse
    return self.bound._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 672, in _parse
    subobj = sc._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 853, in _parse
    obj = self.cases.get(key, self.default)._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 735, in _parse
    subobj = sc._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 1295, in _parse
    return self.bound._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 672, in _parse
    subobj = sc._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 853, in _parse
    obj = self.cases.get(key, self.default)._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 735, in _parse
    subobj = sc._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 1295, in _parse
    return self.bound._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 672, in _parse
    subobj = sc._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 853, in _parse
    obj = self.cases.get(key, self.default)._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 735, in _parse
    subobj = sc._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 672, in _parse
    subobj = sc._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 288, in _parse
    return self._decode(self.subcon._parse(stream, context), context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/adapters.py", line 104, in _decode
    obj, self.subcon.name))
construct.adapters.MappingError: no decoding mapping for 16777216 [matchOp]

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 444, in _parse
    obj.append(self.subcon._parse(stream, context))
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 672, in _parse
    subobj = sc._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 853, in _parse
    obj = self.cases.get(key, self.default)._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 976, in _parse
    obj = self.subcon._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 1295, in _parse
    return self.bound._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 672, in _parse
    subobj = sc._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 1011, in _parse
    return self.subcon._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 853, in _parse
    obj = self.cases.get(key, self.default)._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 672, in _parse
    subobj = sc._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 447, in _parse
    raise ArrayError("expected %d, found %d" % (count, c), sys.exc_info()[1])
construct.core.ArrayError: ('expected 1, found 0', MappingError('no decoding mapping for 16777216 [matchOp]',))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/bob/.local/bin/isign", line 251, in <module>
    isign.resign(app_path, **kwargs)
  File "/home/bob/.local/lib/python3.6/site-packages/isign/isign.py", line 83, in resign
    alternate_entitlements_path)
  File "/home/bob/.local/lib/python3.6/site-packages/isign/archive.py", line 409, in resign
    ua.bundle.resign(deep, signer, provisioning_profile, alternate_entitlements_path)
  File "/home/bob/.local/lib/python3.6/site-packages/isign/bundle.py", line 266, in resign
    super(App, self).resign(deep, signer)
  File "/home/bob/.local/lib/python3.6/site-packages/isign/bundle.py", line 181, in resign
    self.sign(deep, signer)
  File "/home/bob/.local/lib/python3.6/site-packages/isign/bundle.py", line 176, in sign
    executable = self.signable_class(self, self.get_executable_path(), signer)
  File "/home/bob/.local/lib/python3.6/site-packages/isign/signable.py", line 44, in __init__
    self.arches = self._parse_arches()
  File "/home/bob/.local/lib/python3.6/site-packages/isign/signable.py", line 65, in _parse_arches
    self.file_end))
  File "/home/bob/.local/lib/python3.6/site-packages/isign/signable.py", line 99, in _get_arch
    0, self.signer, self.bundle.get_info_prop('CFBundleIdentifier'))
  File "/home/bob/.local/lib/python3.6/site-packages/isign/makesig.py", line 255, in make_signature
    ident)
  File "/home/bob/.local/lib/python3.6/site-packages/isign/makesig.py", line 223, in make_basic_codesig
    return macho_cs.Blob.parse(chunk)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 188, in parse
    return self.parse_stream(BytesIO(data))
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 198, in parse_stream
    return self._parse(stream, Container())
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 1295, in _parse
    return self.bound._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 672, in _parse
    subobj = sc._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 1011, in _parse
    return self.subcon._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 853, in _parse
    obj = self.cases.get(key, self.default)._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 672, in _parse
    subobj = sc._parse(stream, context)
  File "/home/bob/.local/lib/python3.6/site-packages/construct/core.py", line 447, in _parse
    raise ArrayError("expected %d, found %d" % (count, c), sys.exc_info()[1])
construct.core.ArrayError: ('expected 4, found 1', ArrayError('expected 1, found 0', MappingError('no decoding mapping for 16777216 [matchOp]',)))

[rib]

Another thing I forgot to mention is that I've also modified Signable _get_arch() to get it to go down the 'signing from scratch' path:

diff --git a/isign/signable.py b/isign/signable.py
index 6cc2e14..fc2c522 100644
--- a/isign/signable.py
+++ b/isign/signable.py
@@ -82,7 +82,12 @@ class Signable(object, metaclass=ABCMeta):
             self.f.seek(codesig_offset)
             codesig_data = self.f.read(arch['lc_codesig'].data.datasize)
             # log.debug("codesig len: {0}".format(len(codesig_data)))
-        else:
+            codesig = Codesig(self, codesig_data)
+            if len(codesig.get_blobs('CSMAGIC_ENTITLEMENT')) == 0:
+                log.debug('Existing LC_CODE_SIGNATURE missing entitlements')
+                codesig = None
+
+        if codesig == None:
             log.info("signing from scratch!")
             self.sign_from_scratch = True
             entitlements_file = self.bundle.get_entitlements_path()  # '/path/to/some/entitlements.plist'

since I was seeing that even my binaries that haven't been explicitly signed before do still have an LC_CODE_SIGNATURE command but I think it's somehow incomplete for what isign expects. It's able to parse it fine but a missing CSMAGIC_BLOBWRAPPER results in this failure:

Signing with apple_cert: /home/bob/.local/lib/python3.6/site-packages/isign/apple_credentials/applecerts.pem
Signing with key: /home/bob/.isign/key.pem
Signing with certificate: /home/bob/.isign/certificate.pem
Signing with provisioning_profile: /home/bob/.isign/isign.mobileprovision
is_native: True
File <snip/>/Payload/Test.app matched as AppArchive
unarchiving to temp...<snip/>/Payload/Test.app -> /tmp/isign-iraelibc
wrote Entitlements to /tmp/isign-iraelibc/Entitlements.plist
working on /tmp/isign-iraelibc/Test
found thin binary: cputype 16777228, cpusubtype 0
removing ua: /tmp/isign-iraelibc
Traceback (most recent call last):
  File "/home/bob/.local/bin/isign", line 251, in <module>
    isign.resign(app_path, **kwargs)
  File "/home/bob/.local/lib/python3.6/site-packages/isign/isign.py", line 83, in resign
    alternate_entitlements_path)
  File "/home/bob/.local/lib/python3.6/site-packages/isign/archive.py", line 409, in resign
    ua.bundle.resign(deep, signer, provisioning_profile, alternate_entitlements_path)
  File "/home/bob/.local/lib/python3.6/site-packages/isign/bundle.py", line 266, in resign
    super(App, self).resign(deep, signer)
  File "/home/bob/.local/lib/python3.6/site-packages/isign/bundle.py", line 181, in resign
    self.sign(deep, signer)
  File "/home/bob/.local/lib/python3.6/site-packages/isign/bundle.py", line 177, in sign
    executable.sign(self, signer)
  File "/home/bob/.local/lib/python3.6/site-packages/isign/signable.py", line 239, in sign
    offset, new_codesig_data = self._sign_arch(arch, app, signer)
  File "/home/bob/.local/lib/python3.6/site-packages/isign/signable.py", line 125, in _sign_arch
    arch['codesig'].resign(app, signer)
  File "/home/bob/.local/lib/python3.6/site-packages/isign/codesig.py", line 305, in resign
    self.set_signature(signer)
  File "/home/bob/.local/lib/python3.6/site-packages/isign/codesig.py", line 258, in set_signature
    blob_wrappers = self.get_blobs('CSMAGIC_BLOBWRAPPER', min_expected=1, max_expected=1)
  File "/home/bob/.local/lib/python3.6/site-packages/isign/codesig.py", line 94, in get_blobs
    the minimum expected ({})""".format(magic, min_expected))
KeyError: "The number of slots in blob index for magic 'CSMAGIC_BLOBWRAPPER' was less than\n                the minimum expected (1)"

[rib]

Hmm, I just found this pull request against the saucelabs repo that's still open that might help here: sauce-archives#97

[rib]

Although it looks like @ryu2's changes are actually merged in this fork it looks like it has maybe bitrotted somewhat.

It looks like it uses a version of the construct module < 2.8 and the online reference manual no longer documents that version - making it a little awkward to debug. The construct module has then had multiple really-major API breaks for 2.8 and then again for 2.9, including renaming all the integer types and switching to a field-name/type syntax magic. I took a stab at forward porting but basically gave up for now.

Atm I feel like it would almost be easier to re-write this with Python's built in struct parsing api. Based on the history it looks like the construct API is not stable and therefore this feature is currently built on quick sand :/

[rib]

Experimenting with @ryu2's fork and cherry picking my own recent changes I can get it to sign my package from scratch, but then the limitation for me is that the fork is missing the sha256 updates that are in this fork.

[rib]

I bit the bullet and dug into this further and got something that's at least much closer to working but unfortunately still not quite working :/

At least it no longer has any exceptions and comparing hexdumps or pprint_codesig --verbose output I can see that signing from scratch with codesign on OSX vs isign are now really close for me.

Here's what I've experimented with:
https://github.com/rib/isign/tree/wip/rib/sign-from-scratch-for-ios11

It's a very messy commit with lots piled together but in case I do run out of energy it might be useful for someone else.

[ur0]

Hi @rib

I checked out your WIP branch and after a few tweaks to setup.py, it works fine (had 2 successful installs on 11.3 beta).

Here's the diff:

diff --git a/setup.py b/setup.py
index 49e6e1d..442af65 100644
--- a/setup.py
+++ b/setup.py
@@ -31,8 +31,7 @@ setup(
     keywords=['ios', 'app', 'signature', 'codesign', 'sign', 'resign'],
     packages=find_packages(),
     install_requires=[
-        'biplist==0.9',
-        'construct==2.5.5-reupload',
+        'construct==2.5.5',
         'memoizer==0.0.1',
         'pyOpenSSL==17.2.0',
         'biplist==1.0.2'

Thanks for doing this!

[subho007]

@ur0 this is a python3 fork. We use ak-construct not the original construct. ref - https://github.com/appknox/construct

[subho007]

@rib this is an awesome write-up. I am currently debugging your PR.

[subho007]

@rib I can merge the sha256 branch from the apperian fork. You want me to go ahead with that first ?

[rib]

@ur0 i'm not sure about the setup.py change since as far as I saw there was no 2.5.5 version I could install for constructor (via pip) but there was a 2.5.5-reupload. Also it really does depend on biplist so not sure about removing that depends from the list. (I suppose you happen to already have it installed?)

@subho007 actually this branch ended up reverting the python3 porting because I was initially having problems that seemed python3 related, so this branch depends on python 2 again. I think I indirectly figured out the python3 issue while I was debugging this so maybe could revisit this at some point and keep the python3 support.

[ur0]

@rib turns out that setup.py requires two conflicting versions of biplist, so I removed one. Here's how the file looked at ffa6b4.

        'biplist==0.9',
        'construct==2.5.5',
        'memoizer==0.0.1',
        'pyOpenSSL==17.2.0',
        'biplist==1.0.2'

[nicolas17]

Is anyone still working on this? I couldn't get @rib's branch to work.

  File "isign/isign/signable.py", line 100, in _get_arch
    if codesig == None:
UnboundLocalError: local variable 'codesig' referenced before assignment

I added a codesig=None there to trigger the sign-from-scratch code path, then I got this:

Traceback (most recent call last):
  File "./bin/isign", line 253, in <module>
    isign.resign(app_path, **kwargs)
  File "isign/isign/isign.py", line 83, in resign
    alternate_entitlements_path)
  File "isign/isign/archive.py", line 401, in resign
    ua.bundle.resign(deep, signer, provisioning_profile, alternate_entitlements_path)
  File "isign/isign/bundle.py", line 270, in resign
    super(App, self).resign(deep, signer)
  File "isign/isign/bundle.py", line 185, in resign
    self.sign(deep, signer)
  File "isign/isign/bundle.py", line 180, in sign
    executable = self.signable_class(self, self.get_executable_path(), signer)
  File "isign/isign/signable.py", line 46, in __init__
    self.arches = self._parse_arches()
  File "isign/isign/signable.py", line 67, in _parse_arches
    self.file_end))
  File "isign/isign/signable.py", line 108, in _get_arch
    0, self.signer, self.bundle.get_info_prop('CFBundleIdentifier'))
  File "isign/isign/makesig.py", line 283, in make_signature
    ident)
  File "isign/isign/makesig.py", line 176, in make_basic_codesig
    cd1_data = macho_cs.CodeDirectory.build(cd1)
  File "/home/nicolas/.fades/55a32329-888b-4a7a-9e17-74cf1a414a27/local/lib/python2.7/site-packages/construct/core.py", line 212, in build
    self.build_stream(obj, stream)
  File "/home/nicolas/.fades/55a32329-888b-4a7a-9e17-74cf1a414a27/local/lib/python2.7/site-packages/construct/core.py", line 219, in build_stream
    self._build(obj, stream, Container())
  File "/home/nicolas/.fades/55a32329-888b-4a7a-9e17-74cf1a414a27/local/lib/python2.7/site-packages/construct/core.py", line 693, in _build
    sc._build(subobj, stream, context)
  File "/home/nicolas/.fades/55a32329-888b-4a7a-9e17-74cf1a414a27/local/lib/python2.7/site-packages/construct/core.py", line 372, in _build
    raise FieldError(sys.exc_info()[1])
construct.core.FieldError: 'ascii' codec can't decode byte 0xd9 in position 3: ordinal not in range(128)