build: refactor utility functions and meta script

Author: Sam Kleinman

bin/delegated-build

@@ -5,11 +5,11 @@ import sys
 import subprocess
 import argparse
 import datetime
-from docs_meta import PUBLISHED_BRANCHES, get_branch
+from docs_meta import PUBLISHED_BRANCHES
 import utils
 
 branches = PUBLISHED_BRANCHES
-current_branch = get_branch(os.getcwd())
+current_branch = utils.get_branch(os.getcwd())
 
 if current_branch not in branches:
     branches.append(current_branch)
@@ -87,7 +87,7 @@ def update_repo(path, logfile=None, branch='master'):
     change_branch(branch, path)
 
 def build_branch(branch, path, logfile, target='publish', wait=False):
-    if get_branch(path) != branch:
+    if utils.get_branch(path) != branch:
         change_branch(branch, path)
 
     cmd = ['make', target]

bin/docs_meta.py

@@ -1,39 +1,20 @@
 #!/usr/bin/python
 
 import datetime
-import os
-import re
-import subprocess
 import argparse
 import yaml
 
+from utils import write_yaml, shell_value, get_commit, get_branch
+
 # branch and release information source
 GIT_REMOTE = { 'upstream': 'mongodb/docs' }
 MANUAL_BRANCH = 'master'
 PUBLISHED_BRANCHES = [ 'master', 'v2.2' ] # PUBLISHED_BRANCHES **must** be ordered from latest to oldest release.
 PUBLISHED_VERSIONS = [ '2.4', '2.2' ]
+
 STABLE_RELEASE = PUBLISHED_VERSIONS[0]
 UPCOMING_RELEASE = None
 
-def get_build_envs():
-    s = (['build/' + branch + '/branch-source/' for branch in PUBLISHED_BRANCHES ] +
-         ['build/' + branch + '/branch-source-current/' for branch in PUBLISHED_BRANCHES ])
-    s.append('build/' + branch + '/source/')
-    return s
-
-def shell_value(args, path=None):
-    if path is None:
-        path = os.getcwd()
-
-    if isinstance(args , str):
-        r = re.compile("\s+")
-        args = r.split(args)
-
-    p = subprocess.Popen(args, cwd=path, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    r = p.communicate()
-
-    return str(r[0].decode().rstrip())
-
 def get_manual_path():
     branch = get_branch()
 
@@ -44,18 +25,6 @@ def get_manual_path():
 
     return manual_path
 
-def get_commit(path=None):
-    if path is None:
-        path = os.getcwd()
-
-    return shell_value('git rev-parse --verify HEAD', path)
-
-def get_branch(path=None):
-    if path is None:
-        path = os.getcwd()
-
-    return shell_value('git symbolic-ref HEAD', path).split('/')[2]
-
 def get_versions():
     o = []
 
@@ -85,8 +54,7 @@ def output_yaml(fn):
             'pdfs': []
     }
 
-    with open(fn, 'w') as f:
-        f.write(yaml.dump(o, default_flow_style=False))
+    write_yaml(o, fn)
 
 def render_paths(fn):
     paths = {
@@ -101,8 +69,7 @@ def render_paths(fn):
     paths['branch-staging'] = paths['public'] + '/' + get_branch()
 
     if str(fn).endswith('yaml'):
-        with open(fn, 'w') as f:
-            f.write(yaml.dump(paths, default_flow_style=False))
+        utils.write_yaml(paths, fn)
     elif fn == 'print':
         print(yaml.dump(paths, default_flow_style=False))
     else:

bin/makecloth/delegated.py

@@ -6,15 +6,15 @@
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../')))
 
 import utils
-import docs_meta as mongo_meta
+from docs_meta import PUBLISHED_BRANCHES
 
 from makecloth import MakefileCloth
 
 m = MakefileCloth()
 
 def generate_delegated_interface(builders):
-    branches = mongo_meta.PUBLISHED_BRANCHES
-    current_branch = mongo_meta.get_branch()
+    branches = PUBLISHED_BRANCHES
+    current_branch = utils.get_branch()
 
     if current_branch not in branches:
         branches.append(current_branch)

bin/makecloth/json_output.py

@@ -6,15 +6,15 @@
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../')))
 
 import utils
-from docs_meta import get_manual_path, get_branch, render_paths
+from docs_meta import get_manual_path, render_paths
 from makecloth import MakefileCloth
 
 paths = render_paths('dict')
 m = MakefileCloth()
 
 def generate_list_file(outputs, path):
     with open(path, 'w') as f:
-        branch = get_manual_path()
+        branch =get_manual_path()
 
         for fn in outputs:
             url = [ 'http://docs.mongodb.org', branch ]

bin/makecloth/meta.py

@@ -3,18 +3,19 @@
 import os.path
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../')))
 
-import docs_meta as docs_meta
+import utils
+from docs_meta import get_manual_path, MANUAL_BRANCH, render_paths
 from makecloth import MakefileCloth
 m = MakefileCloth()
 
 def generate_meta():
     m.section_break('branch/release meta', block='rel')
-    m.var('manual-branch', docs_meta.MANUAL_BRANCH, block='rel')
-    m.var('current-branch', str(docs_meta.get_branch()), block='rel')
-    m.var('last-commit', str(docs_meta.get_commit()), block='rel')
-    m.var('current-if-not-manual', str(docs_meta.get_manual_path()), block='rel')
+    m.var('manual-branch', MANUAL_BRANCH, block='rel')
+    m.var('current-branch', str(utils.get_branch()), block='rel')
+    m.var('last-commit', str(utils.get_commit()), block='rel')
+    m.var('current-if-not-manual', str(get_manual_path()), block='rel')
 
-    paths = docs_meta.render_paths(True)
+    paths = render_paths(True)
 
     m.section_break('file system paths', block='paths')
     m.var('output', paths['output'], block='paths')

bin/makecloth/pdfs.py

@@ -4,18 +4,18 @@
 import os.path
 
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../')))
-import docs_meta
 import utils
 from makecloth import MakefileCloth
+from docs_meta import render_paths
 
 m = MakefileCloth()
 
-paths = docs_meta.render_paths('dict')
+paths = render_paths('dict')
 
 def pdf_makefile(name, tag):
     name_tagged = '-'.join([name, tag])
     name_tagged_pdf = name_tagged + '.pdf'
-    name_tagged_branch_pdf = '-'.join([name, tag,  docs_meta.get_branch()]) + '.pdf'
+    name_tagged_branch_pdf = '-'.join([name, tag,  utils.get_branch()]) + '.pdf'
 
     generated_latex = '{0}/latex/{1}.tex'.format(paths['branch-output'], name)
     built_tex = '{0}/latex/{1}.tex'.format(paths['branch-output'], name_tagged)

bin/utils.py

@@ -1,9 +1,35 @@
+import re
 import yaml
 import sys
-import os.path
+import os
 import subprocess
 import json
 
+def shell_value(args, path=None):
+    if path is None:
+        path = os.getcwd()
+
+    if isinstance(args , str):
+        r = re.compile("\s+")
+        args = r.split(args)
+
+    p = subprocess.Popen(args, cwd=path, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    r = p.communicate()
+
+    return str(r[0].decode().rstrip())
+
+def get_commit(path=None):
+    if path is None:
+        path = os.getcwd()
+
+    return shell_value('git rev-parse --verify HEAD', path)
+
+def get_branch(path=None):
+    if path is None:
+        path = os.getcwd()
+
+    return shell_value('git symbolic-ref HEAD', path).split('/')[2]
+
 def expand_tree(path, input_extension='yaml'):
     file_list = []
     for root, subFolders, files in os.walk(path):
@@ -40,6 +66,15 @@ def ingest_yaml(filename):
 
     return o
 
+def write_yaml(input, filename):
+    with open(filename, 'w') as f:
+        if isinstance(input, list):
+            f.write(yaml.safe_dump_all(input, default_flow_style=False))
+        elif isinstance(input, dict):
+            f.write(yaml.safe_dump(input, default_flow_style=False))
+        else:
+            raise Exception('cannot dump $s objects to yaml.' % str(type(input)))
+
 def ingest_json(filename):
     o = []
     with open(filename, 'r') as f: