digest authentication - Apache HTTPClient DigestAuth doesn't forward "opaque" value from Challenge

Question

I'm trying to use Digest authentication with HTTP Client against a 3rd-party web service that I don't control.

I started out with the sample code from here:

http://hc.apache.org/httpcomponents-client-4.5.x/httpclient/examples/org/apache/http/examples/client/ClientPreemptiveDigestAuthentication.java 

I got it working against httpbin.org, before attempting the next step described below.

It appears that the target 3rd-party service that I'm using requires the opaque value to be copied from the WWW-Authentication header on the initial response to the Authorization header on the next request, as described here:

https://security.stackexchange.com/questions/24425/what-is-the-opaque-field-in-http-digest-access-authentication-used-for

However, I have turned on wire-logging and stepped through the code (again this is really just the sample code linked above, no need to copy/paste it here) and I see that the opaque is NOT copied.

Any ideas what prevents it from being copied?

I even tried overriding the processChallenge method:

    DigestScheme digestAuth = new DigestScheme() {
        @Override
        public void processChallenge(
                Header header) throws MalformedChallengeException {

but it appears that any value introduced into the Parameters at this point is ignored in the next request.

Answer 1

Finally fixed by overriding the Authorize header explicitly, instead of relying on the internals of HttpClient to do it automatically:

package [...];

import java.util.Arrays;
import java.util.Map;
import java.util.stream.Collectors;

import org.apache.commons.io.IOUtils;
import org.apache.http.*;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.*;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.auth.DigestScheme;
import org.apache.http.impl.client.*;
import org.testng.Assert;

public class DigestTest {

    private static final String URL
            = "https://...";

    private static final String PASSWORD = ...;

    private static final String USER = ...;

    public static void main(String[] args) throws Exception {

        new DigestTest().run();
    }

    public void run() throws Exception {

        HttpGet httpget = new HttpGet(URL);

        HttpHost target
                = new HttpHost(httpget.getURI().getHost(), 443, "https");
        CredentialsProvider credsProvider = new BasicCredentialsProvider();

        UsernamePasswordCredentials credentials
                = new UsernamePasswordCredentials(USER, PASSWORD);
        credsProvider.setCredentials(
                new AuthScope(target.getHostName(), target.getPort()),
                credentials);

        CookieStore cookieStore = new BasicCookieStore();

        CloseableHttpClient httpclient
                = HttpClients.custom().setDefaultCookieStore(cookieStore)
                        .setDefaultCredentialsProvider(credsProvider).build();

        try {

            DigestScheme digestAuth = new DigestScheme();

            digestAuth.overrideParamter("qop", "auth");
            digestAuth.overrideParamter("nc", "0");
            digestAuth.overrideParamter("cnonce", DigestScheme.createCnonce());

            AuthCache authCache = new BasicAuthCache();
            authCache.put(target, digestAuth);

            HttpClientContext localContext = HttpClientContext.create();
            localContext.setAuthCache(authCache);

            CloseableHttpResponse response;

            response = httpclient.execute(target, httpget, localContext);
            Map<String, String> wwwAuth = Arrays
                    .stream(response.getHeaders("WWW-Authenticate")[0]
                            .getElements())
                    .collect(Collectors.toMap(HeaderElement::getName,
                            HeaderElement::getValue));

            // the first call ALWAYS fails with a 401
            Assert.assertEquals(response.getStatusLine().getStatusCode(), 401);

            digestAuth.overrideParamter("opaque", wwwAuth.get("opaque"));
            digestAuth.overrideParamter("nonce", wwwAuth.get("nonce"));
            digestAuth.overrideParamter("realm", wwwAuth.get("Digest realm"));
            Header authenticate = digestAuth.authenticate(credentials, httpget,
                    localContext);
            httpget.addHeader(authenticate);

            response = httpclient.execute(target, httpget, localContext);

            // the 2nd call is the real deal
            Assert.assertEquals(response.getStatusLine().getStatusCode(), 200);

            System.out.println(IOUtils
                    .toString(response.getEntity().getContent(), "utf-8"));

        } finally {
            httpclient.close();
        }
    }

}