8

I'm trying to run a bash script inside GCP Function but somehow it's not working. Here my function, which basically export a file(proxy) to Google Apigee: 

def test2(request):
    cmd = "python ./my-proxy/tools/deploy.py -n myProxy -u userName:!password -o myOrg -e test -d ./my-proxy -p /"
    # no block, it start a sub process.
    p = subprocess.Popen(cmd , shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    # and you can block util the cmd execute finish
    p.wait()
    # or stdout, stderr = p.communicate()
    return "Proxy deployed to Apigee"

Here's my deploy.py file looks like:

!/usr/bin/env python

import base64
import getopt
import httplib
import json
import re
import os
import sys
import StringIO
import urlparse
import xml.dom.minidom
import zipfile


def httpCall(verb, uri, headers, body):
    if httpScheme == 'https':
        conn = httplib.HTTPSConnection(httpHost)
    else:
        conn = httplib.HTTPConnection(httpHost)

    if headers == None:
        hdrs = dict()
    else:
        hdrs = headers

    hdrs['Authorization'] = 'Basic %s' % base64.b64encode(UserPW)
    conn.request(verb, uri, body, hdrs)

    return conn.getresponse()


def getElementText(n):
    c = n.firstChild
    str = StringIO.StringIO()

    while c != None:
        if c.nodeType == xml.dom.Node.TEXT_NODE:
            str.write(c.data)
        c = c.nextSibling

    return str.getvalue().strip()


def getElementVal(n, name):
    c = n.firstChild

    while c != None:
        if c.nodeName == name:
            return getElementText(c)
        c = c.nextSibling

    return None


# Return TRUE if any component of the file path contains a directory name that
# starts with a "." like '.svn', but not '.' or '..'
def pathContainsDot(p):
    c = re.compile('\.\w+')

    for pc in p.split('/'):
        if c.match(pc) != None:
            return True

    return False


def getDeployments():
    # Print info on deployments
    hdrs = {'Accept': 'application/xml'}
    resp = httpCall('GET',
            '/v1/organizations/%s/apis/%s/deployments' \
                % (Organization, Name),
            hdrs, None)

    if resp.status != 200:
        return None

    ret = list()
    deployments = xml.dom.minidom.parse(resp)
    environments = deployments.getElementsByTagName('Environment')

    for env in environments:
        envName = env.getAttribute('name')
        revisions = env.getElementsByTagName('Revision')
        for rev in revisions:
            revNum = int(rev.getAttribute('name'))
            error = None
            state = getElementVal(rev, 'State')
            basePaths = rev.getElementsByTagName('BasePath')

            if len(basePaths) > 0:
                basePath = getElementText(basePaths[0])
            else:
                basePath = 'unknown'

            # svrs = rev.getElementsByTagName('Server')
            status = {'environment': envName,
                    'revision': revNum,
                    'basePath': basePath,
                    'state': state}

            if error != None:
                status['error'] = error

            ret.append(status)

    return ret


def printDeployments(dep):
    for d in dep:
        print 'Environment: %s' % d['environment']
        print '  Revision: %i BasePath = %s' % (d['revision'], d['basePath'])
        print '  State: %s' % d['state']
        if 'error' in d:
            print '  Error: %s' % d['error']

ApigeeHost = 'https://api.enterprise.apigee.com'
UserPW = None
Directory = None
Organization = None
Environment = None
Name = None
BasePath = '/'
ShouldDeploy = True

Options = 'h:u:d:e:n:p:o:i:z:'

opts = getopt.getopt(sys.argv[1:], Options)[0]

for o in opts:
    if o[0] == '-n':
        Name = o[1]
    elif o[0] == '-o':
        Organization = o[1]
    elif o[0] == '-h':
        ApigeeHost = o[1]
    elif o[0] == '-d':
        Directory = o[1]
    elif o[0] == '-e':
        Environment = o[1]
    elif o[0] == '-p':
        BasePath = o[1]
    elif o[0] == '-u':
        UserPW = o[1]
    elif o[0] == '-i':
        ShouldDeploy = False
    elif o[0] == '-z':
        ZipFile = o[1]

if UserPW == None or \
        (Directory == None and ZipFile == None) or \
        Environment == None or \
        Name == None or \
        Organization == None:
    print """Usage: deploy -n [name] (-d [directory name] | -z [zipfile])
              -e [environment] -u [username:password] -o [organization]
              [-p [base path] -h [apigee API url] -i]
    base path defaults to "/"
    Apigee URL defaults to "https://api.enterprise.apigee.com"
    -i denotes to import only and not actually deploy
    """
    sys.exit(1)

url = urlparse.urlparse(ApigeeHost)
httpScheme = url[0]
httpHost = url[1]

body = None

if Directory != None:
    # Construct a ZIPped copy of the bundle in memory
    tf = StringIO.StringIO()
    zipout = zipfile.ZipFile(tf, 'w')

    dirList = os.walk(Directory)
    for dirEntry in dirList:
        if not pathContainsDot(dirEntry[0]):
            for fileEntry in dirEntry[2]:
                if not fileEntry.endswith('~'):
                    fn = os.path.join(dirEntry[0], fileEntry)
                    en = os.path.join(
                            os.path.relpath(dirEntry[0], Directory),
                            fileEntry)
                    print 'Writing %s to %s' % (fn, en)
                    zipout.write(fn, en)

    zipout.close()
    body = tf.getvalue()
elif ZipFile != None:
    f = open(ZipFile, 'r')
    body = f.read()
    f.close()

# Upload the bundle to the API
hdrs = {'Content-Type': 'application/octet-stream',
        'Accept': 'application/json'}
uri = '/v1/organizations/%s/apis?action=import&name=%s' % \
            (Organization, Name)
resp = httpCall('POST', uri, hdrs, body)

if resp.status != 200 and resp.status != 201:
    print 'Import failed to %s with status %i:\n%s' % \
            (uri, resp.status, resp.read())
    sys.exit(2)

deployment = json.load(resp)
revision = int(deployment['revision'])

print 'Imported new proxy version %i' % revision

if ShouldDeploy:
    # Undeploy duplicates
    deps = getDeployments()
    for d in deps:
        if d['environment'] == Environment and \
            d['basePath'] == BasePath and \
            d['revision'] != revision:
            print 'Undeploying revision %i in same environment and path:' % \
                    d['revision']
            conn = httplib.HTTPSConnection(httpHost)
            resp = httpCall('POST',
                    ('/v1/organizations/%s/apis/%s/deployments' +
                            '?action=undeploy' +
                            '&env=%s' +
                            '&revision=%i') % \
                        (Organization, Name, Environment, d['revision']),
                 None, None)
            if resp.status != 200 and resp.status != 204:
                print 'Error %i on undeployment:\n%s' % \
                        (resp.status, resp.read())

    # Deploy the bundle
    hdrs = {'Accept': 'application/json'}
    resp = httpCall('POST',
        ('/v1/organizations/%s/apis/%s/deployments' +
                '?action=deploy' +
                '&env=%s' +
                '&revision=%i' +
                '&basepath=%s') % \
            (Organization, Name, Environment, revision, BasePath),
        hdrs, None)

    if resp.status != 200 and resp.status != 201:
        print 'Deploy failed with status %i:\n%s' % (resp.status, resp.read())
        sys.exit(2)

deps = getDeployments()
printDeployments(deps)

This works when I run locally on my machine, but not on GCP. Don't know if have anything to do with the fact I'm connecting to Google Apigee with this function. It's weird that the logs on GCP doesn't show any error, however I don't have my proxy exported to Apigee.

Thanks the help!

UPDATED: tried using subprocess.check_output() as encouraged by some here: 

def test(request):
    output = None
    try:
        output = subprocess.check_output([
        "./my-proxy/tools/deploy.py", 
        '-n',  'myProxy',
        '-u', 'myUserName:myPassword',
        '-o', 'myOrgName',
        '-e', 'test',
        '-d', './my-proxy',
        '-p', '/'])

    except:
        print(output)    

    return output

And still not working on GCP. Like I mentioned before, it works like a charm (both solutions above) in my machine, but in GCP doesn't. As you can see from the image below, I get a 200 after executing deploy.py from GCP but my file doesn't go to Apigee:

enter image description here

GCP logs doesn't show any error as well:

enter image description here

Rafael Paz
  • 437
  • 6
  • 17
  • Have you considered simply invoking python code within the python function itself, instead of trying to launch a subprocess? – Doug Stevenson Mar 01 '19 at 04:50
  • thanks for the response @DougStevenson, I'm pretty new to Python, would be great to give a try, if not asking to much would you have an example? Cheers – Rafael Paz Mar 01 '19 at 04:51
  • I don't have an example. I've just never heard of anyone trying to subprocess another python process in Cloud Functions. Maybe it just doesn't work, and was never intended to work. – Doug Stevenson Mar 01 '19 at 04:59
  • No worries @DougStevenson thanks for the help anyway. Will keep up see if can sort this out. – Rafael Paz Mar 01 '19 at 05:06
  • 1
    Envisage your function as being deployed into a container that primarily includes (in this case) a Python runtime and *doesn't* include a shell. – DazWilkin Mar 01 '19 at 05:16
  • Pretty new to GCP as well, trying to read as much as I can, but didn't know about that. Thanks @DazWilkin for let me know about that. Would you have any suggestion on how I could sort a way to call the deploy.py file from my function? Still don't know what to do with it. – Rafael Paz Mar 01 '19 at 05:24
  • what I think it's weird is the fact that don't get any error when calling the function. – Rafael Paz Mar 01 '19 at 05:26
  • As @doug-stevenson suggested, rather than have Python invoke a shell that invokes Python, you may want to call the relevant function from `deploy.py` directly from your test function. I don't understand sufficiently what you're trying to do nor how Apigee works so it's difficult to provide precise guidance. – DazWilkin Mar 01 '19 at 05:44
  • I think that calling the relevant function from deploy.py in my test function won't be possible to do as the deploy.py file invokes a series of bash command inside of it: "read, echo, curl etc.." So even if I call the function inside the deploy.py, shell commands will be executed inside the file. – Rafael Paz Mar 01 '19 at 05:53
  • Apigee is a Google Cloud Plaftform, they work with the concept of API proxies (don't wanna get too much on it as will go away from the purpose of the question). So basically what I'm trying to do is to connect from my function on GCP to Apigee. Everytime my function is triggered I connect to my Apigee account passing a file (API proxy) which will be committed from time to time. Apigee provided this deploy.py file in order to do that, so that's what I'm trying to achieve here. – Rafael Paz Mar 01 '19 at 05:57
  • Can you try this with [`subprocess.check_output()`](https://docs.python.org/3/library/subprocess.html#subprocess.check_output) instead? It's likely that output from the `deploy.py` is not being redirected correctly. – Dustin Ingram Mar 01 '19 at 21:42
  • Also note that the first argument to [`subprocess.Popen()`](https://docs.python.org/3/library/subprocess.html#popen-constructor) should be a list of arguments, not a single string. This might be the root of the problem. – Dustin Ingram Mar 01 '19 at 21:44
  • @DustinIngram thanks for the help mate, much appreciated for you taking your time to help a stranger :), tried using subprocess.check.output as you mentioned it worked on my machine just like subprocess.Popen() worked as well, however it doesn't work on GCP. Please read my question again, updated the question. Is it a bug on GCP? – Rafael Paz Mar 02 '19 at 03:54
  • @RafaelPaz Not sure if it suits your case, but you could deploy your shell script to cloud run and call it from the cloud function. – Rafael Lemos Sep 29 '20 at 12:52

1 Answers1

3

This is possible!

The python executable is not installed or linked in the Cloud Function runtime, but python3 is. Hence, there are a few ways to solve this:

  1. specify python3 as the program to run: "python3 ./my-proxy/tools/deploy.py ...";

  2. add the #! operator in the deploy.py script: #!/usr/bin/env python3;

  3. specify the python interpreter to Popen. You can use sys.executable to refer to the currently used executable:

     process = subprocess.Popen(
     [
         sys.executable,
         "./deploy.py",
         "-n",
         "myProxy",
         "-u",
         "myUserName:myPassword",
         "-o",
         "myOrgName",
         "-e",
         "test",
         "-d",
         "./my-proxy",
         "-p",
         "/",
     ],
     stdout=subprocess.PIPE,
     stderr=subprocess.PIPE,
     universal_newlines=True,
 )

You did not see an error because it was generated in a subprocess, printed to its stderr, and subsequently captured by your program with process.communicate() or process.check_output(...), but not printed. To see the error you are experiencing, you can print out the content of stdout and stderr:

    out, err = process.communicate()
    log.debug("returncode = %s", process.returncode)
    log.debug("stdout = %s", out)
    log.debug("stderr = %s", err)

Check out our source code we used to analyze, reproduce and solve your question on github

Stendert
  • 87
  • 7