hash - OpenSSL: how to get x509 -subject_hash "manually"?

Question

I'm trying to get the same result as

openssl X509 -in certificate.pem -subject_hash

without using the function directly but instead by extracting the cerrtificate name and building the propper canonical representation to then take the sha-1 hash from. Is this possible? Something like

openssl x509 -in certificate.pem -subject -noout -nameopt dn_rev -nameopt RFC2253 | tr '[:upper:]' '[:lower:]' | openssl dgst -sha1 -binary | xxd -p

and then truncating the hash to 4bytes or so... so far I didn't get it though

$ openssl x509 -in .pem -subject_hash -noout
cc952886

$openssl x509 -in certificate.pem -subject -noout -nameopt RFC2253 | tr '[:upper:]' '[:lower:]' | openssl dgst -sha1 -binary | xxd -p
0b6a015b2a7ed2a5f3695f1d46a0c20006de300a

The respective c code is here: https://github.com/openssl/openssl/blob/d53b437f9992f974c1623e9b9b9bdf053aefbcc3/crypto/x509/x509_cmp.c#L261

unsigned long X509_NAME_hash_ex(const X509_NAME *x, OSSL_LIB_CTX *libctx,
                                const char *propq, int *ok)
{
    unsigned long ret = 0;
    unsigned char md[SHA_DIGEST_LENGTH];
    EVP_MD *sha1 = EVP_MD_fetch(libctx, "SHA1", propq);

    /* Make sure X509_NAME structure contains valid cached encoding */
    i2d_X509_NAME(x, NULL);
    if (ok != NULL)
        *ok = 0;
    if (sha1 != NULL
        && EVP_Digest(x->canon_enc, x->canon_enclen, md, NULL, sha1, NULL)) {
        ret = (((unsigned long)md[0]) | ((unsigned long)md[1] << 8L) |
               ((unsigned long)md[2] << 16L) | ((unsigned long)md[3] << 24L)
               ) & 0xffffffffL;
        if (ok != NULL)
            *ok = 1;
    }
    EVP_MD_free(sha1);
    return ret;
}

and the canonical representation of the name is set here: https://github.com/openssl/openssl/blob/256d41d4371720ccfe1a4fead6bd28ed5071bcdd/crypto/x509/x_name.c#L303

/*
 * This function generates the canonical encoding of the Name structure. In
 * it all strings are converted to UTF8, leading, trailing and multiple
 * spaces collapsed, converted to lower case and the leading SEQUENCE header
 * removed. In future we could also normalize the UTF8 too. By doing this
 * comparison of Name structures can be rapidly performed by just using
 * memcmp() of the canonical encoding. By omitting the leading SEQUENCE name
 * constraints of type dirName can also be checked with a simple memcmp().
 */

static int x509_name_canon(X509_NAME *a)
{
    unsigned char *p;
    STACK_OF(STACK_OF_X509_NAME_ENTRY) *intname;
    STACK_OF(X509_NAME_ENTRY) *entries = NULL;
    X509_NAME_ENTRY *entry, *tmpentry = NULL;
    int i, set = -1, ret = 0, len;

    OPENSSL_free(a->canon_enc);
    a->canon_enc = NULL;
    /* Special case: empty X509_NAME => null encoding */
    if (sk_X509_NAME_ENTRY_num(a->entries) == 0) {
        a->canon_enclen = 0;
        return 1;
    }
    intname = sk_STACK_OF_X509_NAME_ENTRY_new_null();
    if (intname == NULL) {
        ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE);
        goto err;
    }
    for (i = 0; i < sk_X509_NAME_ENTRY_num(a->entries); i++) {
        entry = sk_X509_NAME_ENTRY_value(a->entries, i);
        if (entry->set != set) {
            entries = sk_X509_NAME_ENTRY_new_null();
            if (entries == NULL)
                goto err;
            if (!sk_STACK_OF_X509_NAME_ENTRY_push(intname, entries)) {
                sk_X509_NAME_ENTRY_free(entries);
                ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE);
                goto err;
            }
            set = entry->set;
        }
        tmpentry = X509_NAME_ENTRY_new();
        if (tmpentry == NULL) {
            ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE);
            goto err;
        }
        tmpentry->object = OBJ_dup(entry->object);
        if (tmpentry->object == NULL) {
            ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE);
            goto err;
        }
        if (!asn1_string_canon(tmpentry->value, entry->value))
            goto err;
        if (!sk_X509_NAME_ENTRY_push(entries, tmpentry)) {
            ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE);
            goto err;
        }
        tmpentry = NULL;
    }

    /* Finally generate encoding */
    len = i2d_name_canon(intname, NULL);
    if (len < 0)
        goto err;
    a->canon_enclen = len;

    p = OPENSSL_malloc(a->canon_enclen);
    if (p == NULL) {
        ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE);
        goto err;
    }

    a->canon_enc = p;

    i2d_name_canon(intname, &p);

    ret = 1;

 err:
    X509_NAME_ENTRY_free(tmpentry);
    sk_STACK_OF_X509_NAME_ENTRY_pop_free(intname,
                                         local_sk_X509_NAME_ENTRY_pop_free);
    return ret;
}

question from:https://stackoverflow.com/questions/66055956/openssl-how-to-get-x509-subject-hash-manually

Answer 1

Waitting for answers