Running CURL through Java: Windows and Linux Executions

Normally, the environment variables are set through envp and the command is broken up into separate strings.

However, when on Windows, all the envp and command strings should be concatenated and then processed through Git Bash.  Here is the implementation.

package com.mycompany.util;

import java.io.BufferedReader;

import java.io.BufferedWriter;

import java.io.File;

import java.io.FileReader;

import java.io.FileWriter;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.io.OutputStream;

import java.io.PrintWriter;

public class CurlHelperGeneric {

    

    /**
     *  USERNAME=sizu
        RSA_TOKEN=myRSAToken
        http_proxy=http://myproxy.com:8080/ https_proxy=http://myproxy.com:8080/ curl --proxy-header "Proxy-Authorization: Basic $(echo -n "USERNAME:$RSA_TOKEN" | openssl enc -base64 -e)" "https://myservice.com/registry/v1/service" -k 

        http_proxy=http://myproxy.com:8080/ https_proxy=http://myproxy.com:8080/ curl --proxy-header 'Proxy-Authorization: "+helper.getBasic()+"' 'https://myservice.com/registry/v1/service' -k
     * @param args
     * @throws Exception
     */

    public static void main (String[] args) throws Exception {

         

        CurlHelperGeneric helper = new CurlHelperGeneric();

        String[] cmdArray = new String[] {

                "curl", 

                "--proxy-header", 

                "Proxy-Authorization: Basic c2lXXXXXXzEyMjAwMjQyMTI2OA==",

                "https://myservice.com/v1/pools",
                "-k"
                };
        String[] envp = new String[]{
            "http_proxy=http://myproxy.com:8080/",
            "https_proxy=http://myproxy.com:8080/"
        };
        
        String output = helper.runCommand(cmdArray, envp);
        
        System.out.println("Output:"+output);
    }
    
    public String runCommand(String[] cmdArray, String[] envp) {
        String os = System.getProperty("os.name");
        if(os.toUpperCase().contains("WINDOWS")) {
//            String[] cmdArray = new String[] {
//                    "C:\\Users\\sizu\\AppData\\Local\\Programs\\Git\\bin\\sh.exe",
//                    "--login",
//                    "-i",
//                    "-c",
//                    "http_proxy=http://myproxy.com:8080/ https_proxy=http://myproxy.com:8080/ curl --proxy-header 'Proxy-Authorization: "+proxyAuthorization+"' '"+httpString+"' -k",
//            };
//            String[] envp = new String[]{};
            
            String curlCommandToRunThroughGit = "";
            for(int i=0; i<envp.length; i++) {
                curlCommandToRunThroughGit = curlCommandToRunThroughGit + envp[i] + " ";
            }
            for(int i=0; i<cmdArray.length; i++) {
                curlCommandToRunThroughGit = curlCommandToRunThroughGit + "'" + cmdArray[i] + "' "; // Wrap Authorization and https String with single quotes
            }
            
            cmdArray = new String[] {
                "C:\\Users\\sizu\\AppData\\Local\\Programs\\Git\\bin\\sh.exe",
                "--login",
                "-i",
                "-c",
                curlCommandToRunThroughGit,
            };
            envp = new String[]{};
        }
        
        
        
        try {
            Process process = Runtime.getRuntime().exec(cmdArray, envp);
            StreamGobbler errorGobbler = new StreamGobbler(process.getErrorStream()); // Gobble Stream even if not used
            StreamGobbler outputGobbler = new StreamGobbler(process.getInputStream());
            errorGobbler.start();
            outputGobbler.start();
            int processComplete = process.waitFor(); // Wait for process to finish    
            
//            System.out.println("ERR:"+errorGobbler.streamContent.toString());
//            System.out.println("OUT:"+outputGobbler.streamContent.toString());
            
            return outputGobbler.streamContent.toString();
        } catch (Exception ex) {
            ex.printStackTrace();
            return null;
        }
    }
    
    public static class StreamGobbler extends Thread {
        InputStream is;
        
        StreamGobbler(InputStream is) {
            this.is = is;
        }
        
        StringBuilder streamContent = new StringBuilder();
        
        public void run() {
            try {
                InputStreamReader isr = new InputStreamReader(is);
                BufferedReader br = new BufferedReader(isr);
                String line=null;
                while ( (line = br.readLine()) != null) {
                    streamContent.append(line);
                }
            } catch (IOException ioe) {
                ioe.printStackTrace();  
            }
        }
    }
}

Code Formatted by ToGoTutor

Running CURL Through Java

Sometimes, for whatever reason, curl and Java will not execute the same way.  In some cases, a Java request will work, while in other cases a curl request will work.  Or, you may know how to make one use case work but not the other.

package com.mycompany.util;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintWriter;

public class CurlHelper {
    public String gitExecutable = "C:\\Users\\sizu\\AppData\\Local\\Programs\\Git\\bin\\sh.exe";
    public String fileToRunCurl = "C:\\command.bat";
    public String outputFile = "C:\\command.txt";
    public String curlCommand;

    private String fullCommand;
    private String output = null;
 
    public static void main (String[] args) throws Exception {
         
        CurlHelper helper = new CurlHelper ();
        helper.curlCommand = "http_proxy=http://myproxy.mycompany.com:8080/ https_proxy=http://myproxy.mycompany.com:8080/ curl --proxy-header 'Proxy-Authorization: Basic c2XXXXXwMzEyMjAwMjIyMDkzOA==' 'https://myapiendpoint.mycompany.com:8080/solr/prod/select?rows=10000&format=json' -k";
        
        System.out.println("Output:"+helper.runCurlCommandViaGitBash());
    }
    
    public String runCurlCommandViaGitBash() {
        fullCommand = "\""+gitExecutable+"\" --login -i -c \""+curlCommand+" > '"+outputFile+"'\"";
        createCommandFile();
        executeCommand();
        readOutputFile();
        
        return output;
    }
    
    private void createCommandFile() {
        // Step 1: Write command in command.bat
        File fileToCreate = new File(fileToRunCurl);
        try {
            if(fileToCreate.getParentFile() != null && !fileToCreate.exists()) {
                fileToCreate.getParentFile().mkdirs();
            }
            FileWriter fw = new FileWriter(fileToCreate);
            BufferedWriter bw = new BufferedWriter(fw);
            bw.write(fullCommand);
            bw.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    
    private void executeCommand() {
        System.out.println("executeCommand START");
        // Step 2: Execute command.bat
        try {
            String[] cmdArray = new String[1];
            cmdArray[0] = fileToRunCurl;
    
            Process process = Runtime.getRuntime().exec(cmdArray, null, new File("C:\\"));
            System.out.println("executeCommand B");
            
            // WHEN RUNNING RUNTIME EXEC MULTIPLE TIMES (Over 1000 Runs) WOULD FREEZE - CLEAR ERROR AND OUTPUT STREAMS: https://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
            // any error message?
            StreamGobbler errorGobbler = new StreamGobbler(process.getErrorStream(), "ERR");            
            
            // any output?
            StreamGobbler outputGobbler = new StreamGobbler(process.getInputStream(), "OUT");
                
            // kick them off
            errorGobbler.start();
            outputGobbler.start();
            // WHEN RUNNING RUNTIME EXEC MULTIPLE TIMES (Over 1000 Runs) WOULD FREEZE - CLEAR ERROR AND OUTPUT STREAMS: https://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
            
            int processComplete = process.waitFor(); // Sometimes the process would never complete.  // Could add logic to monitor ERR stream and call process.destroy() if necessary.
        } catch (Exception e) {
            e.printStackTrace();
        }
        System.out.println("executeCommand FINISH");
    }
    
    private void readOutputFile() {
        output = "";
        try {
            FileReader fr = new FileReader(new File(outputFile));
            BufferedReader br = new BufferedReader(fr);
            String sCurrentLine;
            while ((sCurrentLine = br.readLine()) != null) {
                output = output + sCurrentLine + '\n';
                //System.out.println(sCurrentLine);
            }
            br.close();
            fr.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    
    public static class StreamGobbler extends Thread
    {
        InputStream is;
        String type;
        OutputStream os;
        
        StreamGobbler(InputStream is, String type)
        {
            this(is, type, null);
        }
        StreamGobbler(InputStream is, String type, OutputStream redirect)
        {
            this.is = is;
            this.type = type;
            this.os = redirect;
        }
        
        public void run()
        {
            try
            {
                PrintWriter pw = null;
                if (os != null)
                    pw = new PrintWriter(os);
                    
                InputStreamReader isr = new InputStreamReader(is);
                BufferedReader br = new BufferedReader(isr);
                String line=null;
                while ( (line = br.readLine()) != null)
                {
                    if (pw != null)
                        pw.println(line);
                    System.out.println(type + ">" + line);    
                }
                if (pw != null)
                    pw.flush();
            } catch (IOException ioe)
                {
                ioe.printStackTrace();  
                }
        }
    }
}

Code Formatted by ToGoTutor

This code give credit to www.javaworld.com for creating the Stream Gobbler, without which the process would block for for loops, running this curl helper thousands of times. The Stream Gobbler, makes sure the error and output streams do not get clogged up.

Sometimes using http proxies might be easier in curl than java.  Also, you might have rsa tokens that need to be Base 64 encoded to access the proxy.  Below will only work if USERNAME and RSA_TOKEN are system variables.

http_proxy=http://myproxy.mycompany.com:8080/ https_proxy=http://myproxy.mycompany.com:8080/ curl --proxy-header "Proxy-Authorization: Basic $(echo -n "$USERNAME:$RSA_TOKEN" | openssl enc -base64 -e)" "https://myservice.com/registry/v1/service" -k

Code Formatted by ToGoTutor

Or you might encode the Authorization in Java, prior to using the CurlHelper above.

public static String getBasicFromUsernameAndRSAToken() {
        String basic_auth = new String(org.apache.commons.codec.binary.Base64.encodeBase64(("USERNAME:RSA_TOKEN").getBytes()));
        return "Basic " + basic_auth;
    }

Code Formatted by ToGoTutor

In some cases, you will have proxy authentication and authentication.

"curl 'https://myservice.mycompany.com/api/query/Applications' -H 'Authorization: ll/V1nJzd3kl4q9Y/ln7XXXXXX2sInDmAKnSzp6HMSon0YlfDMXNbJrUw9OWDmL'"

Code Formatted by ToGoTutor