gh-repo-export

#!/usr/bin/env bash
# shellcheck disable=SC2213,SC2214,SC2155
set -e

ARCHIVE=
ARCHIVE_PER_REPO=false
EXCLUDE_ATTACHMENTS=false
EXCLUDE_GIT_DATA=false
EXCLUDE_METADATA=false
EXCLUDE_OWNER_PROJECTS=false
EXCLUDE_RELEASES=false
LOCK_REPOSITORIES=false
SKIP_DOWNLOAD=false

__USAGE="
Bulk exports a list of Git repositories from an organization

USAGE
  gh repo-export [flags] <organization> <repo1> <repo2> ... <repoN>
  gh repo-export [flags] <organization> <path/to/repos file>

FLAGS
      --archive=string                Name of archive excluding extension; default 'migration-archive-<migration_id>'
      --archive-per-repo              Generates 1 archive per repository instead of 1 archive for all repositories
  -d, --debug                         Enable debugging
      --exclude-attachments           Indicates attachments should be excluded from the migration
      --exclude-git-data              Indicates git data should be excluded from the migration
      --exclude-metadata              Indicates metadata should be excluded from the migration
      --exclude-owner-projects        Indicates projects owned by the organization or users should be excluded from the migration
      --exclude-releases              Indicates releases should be excluded from the migration
  -h, --help                          Display help usage
      --hostname=string               Hostname of the GitHub instance to authenticate with
      --lock-repositories             Indicates repositories should be locked (to prevent manipulation) while migrating data
      --skip-download                 Skip downloading the archive(s), instead outputting the URL(s) to stdout (curl and sed required)

EXAMPLES
  # Generate 1 archive containing test1 repository, reading repositories from arguments
  $ gh repo-export tinyfists test1

  # Generate 1 archive containing both test1 and test2 repositories, reading repositories from arguments
  $ gh repo-export tinyfists test1 test2

  # Generate 1 archive containing both test1 and test2 repositories, reading repositories from arguments, and output the archive URL instead of downloading
  $ gh repo-export --skip-download tinyfists test1 test2

  # Generate 2 archives, 1 containing test1 repository and 1 containing test2 repository, reading repositories from arguments
  $ gh repo-export --archive-per-repo tinyfists test1 test2

  # Generate 2 archives, 1 containing test1 repository and 1 containing test2 repository, reading repositories from 'repos.txt' file
  $ gh repo-export --archive-per-repo tinyfists repos.txt
";

die() {
  printf "\nError: %s\n" "$1"
  echo "$__USAGE"
  exit 1
}

if ! type -p gh > /dev/null; then
	die "'gh' could not be found"
fi

if ! type -p jq > /dev/null; then
	die "'jq' could not be found"
fi

while getopts "dh-:" OPT; do
  if [ "$OPT" = "-" ]; then     # long option: reformulate OPT and OPTARG
    OPT="${OPTARG%%=*}"         # extract long option name
    OPTARG="${OPTARG#"$OPT"}"   # extract long option argument (may be empty)
    OPTARG="${OPTARG#=}"        # if long option argument, remove assigning `=`
  fi

  case "$OPT" in
    archive)
      ARCHIVE="${OPTARG}"
      ;;

    archive-per-repo)
      ARCHIVE_PER_REPO=true
      ;;

    debug | d)
      set -x
      ;;

    exclude-attachments)
      EXCLUDE_ATTACHMENTS=true
      ;;

    exclude-git-data)
      EXCLUDE_GIT_DATA=true
      ;;

    exclude-metadata)
      EXCLUDE_METADATA=true
      ;;

    exclude-owner-projects)
      EXCLUDE_OWNER_PROJECTS=true
      ;;

    exclude-releases)
      EXCLUDE_RELEASES=true
      ;;

    help | h)
      echo "$__USAGE"
      exit 0
      ;;

    hostname)
      export GH_HOST="${OPTARG}"
      ;;

    lock-repositories)
      LOCK_REPOSITORIES=true
      ;;

    skip-download)
      SKIP_DOWNLOAD=true
      ;;

    *)
      die "Unknown flag '$OPT'"
      ;;
  esac
done

# shift so that $@, $1, etc. refer to the non-option arguments
shift "$((OPTIND-1))"

if test "$#" -lt 2; then
  die "Must provide organization and repositories arguments"
fi

# Process arguments for organization and repository information
ORGANIZATION="$1"
declare -a REPOSITORIES

if [[ "$#" -eq 2 && -f "$2" ]]; then
  # If argument is valid file, read repositories from file
  REPOSITORIES_FILE="$2"
  printf "Reading repositories from file: %s\n" "$REPOSITORIES_FILE"

  while IFS= read -r REPOSITORY; do
    REPOSITORIES+=("$REPOSITORY")
  done < "$REPOSITORIES_FILE"
else
  # Otherwise, assume arguments are repositories
  REPOSITORIES=("${@:2}")
  printf "Reading repositories from arguments: %s\n" "${REPOSITORIES[*]}"
fi

# Start migration of specified GitHub repositories
declare -a MIGRATIONS

start_migration() {
  local REPOSITORIES=("$@")
  # shellcheck disable=SC2016
  local MIGRATION_JQ='{
    "lock_repositories": $lock_repositories,
    "exclude_attachments": $exclude_attachments,
    "exclude_git_data": $exclude_git_data,
    "exclude_metadata": $exclude_metadata,
    "exclude_owner_projects": $exclude_owner_projects,
    "exclude_releases": $exclude_releases,
    "repositories": $ARGS.positional,
  }'

  local MIGRATION_INPUTS=$(
    jq -n "$MIGRATION_JQ" \
      --argjson exclude_attachments "$EXCLUDE_ATTACHMENTS" \
      --argjson exclude_git_data "$EXCLUDE_GIT_DATA" \
      --argjson exclude_metadata "$EXCLUDE_METADATA" \
      --argjson exclude_owner_projects "$EXCLUDE_OWNER_PROJECTS" \
      --argjson exclude_releases "$EXCLUDE_RELEASES" \
      --argjson lock_repositories "$LOCK_REPOSITORIES" \
      --args "${REPOSITORIES[@]}")


  local MIGRATION_ID=$(echo "$MIGRATION_INPUTS" | gh api "/orgs/$ORGANIZATION/migrations" -X POST -p wyandotte --input - --jq '.id')
  printf "Starting migration %s for repositories: %s\n" "$MIGRATION_ID" "${REPOSITORIES[*]}"
  # shellcheck disable=SC2004
  MIGRATIONS[$MIGRATION_ID]="pending"
}

if $SKIP_DOWNLOAD; then
  if ! type -p curl >/dev/null; then
    die "'curl' is required for --skip-download option and could not be found"
  fi

  if ! type -p sed >/dev/null; then
    die "'sed' is required for --skip-download option and could not be found"
  fi

  if [[ $ARCHIVE != "" ]]; then
    die "--archive cannot be used with --skip-download"
  fi
fi

if $ARCHIVE_PER_REPO; then
  for REPOSITORY in "${REPOSITORIES[@]}"; do
    start_migration "$REPOSITORY"
  done
else
  start_migration "${REPOSITORIES[@]}"
fi

# Watch migrations that are waiting to run or complete, sleeping periodically to give them time and avoid rate limiting
is_waiting() {
  local MIGRATION_ID="$1"
  local MIGRATION_STATE="${MIGRATIONS[$MIGRATION_ID]}"

  [[ "$MIGRATION_STATE" == "pending" || "$MIGRATION_STATE" == "exporting" ]]
}

MIGRATION_WATCH=true

while $MIGRATION_WATCH; do
  MIGRATION_WATCH=false

  for MIGRATION_ID in "${!MIGRATIONS[@]}"; do
    if is_waiting "$MIGRATION_ID"; then
      # shellcheck disable=SC2004
      MIGRATIONS[$MIGRATION_ID]=$(gh api "/orgs/$ORGANIZATION/migrations/$MIGRATION_ID" -p wyandotte --jq '.state')
      printf "Watching migration %s with '%s' state\n" "$MIGRATION_ID" "${MIGRATIONS[$MIGRATION_ID]}"

      if is_waiting "$MIGRATION_ID"; then
        MIGRATION_WATCH=true
      fi
    fi
  done

  if $MIGRATION_WATCH; then
    sleep 15
  fi
done

# Download migration archives based upon how repositories were bundled
#
# If all repositories in 1 migration archive, we can allow user to fully specify archive name.
# Otherwise, we need to differentiate with migration IDs
download_migration() {
  local MIGRATION_ID="$1"
  local MIGRATION_FILE="$2"
  local MIGRATION_STATE="${MIGRATIONS[$MIGRATION_ID]}"

  if [[ "$MIGRATION_STATE" == "exported" ]]; then
    printf "Downloading migration %s archive to %s\n" "$MIGRATION_ID" "$MIGRATION_FILE"
    gh api "/orgs/$ORGANIZATION/migrations/$MIGRATION_ID/archive" -p wyandotte > "$MIGRATION_FILE"
  else
    printf "Skipping migration %s due to '%s' state; please investigate\n" "$MIGRATION_ID" "$MIGRATION_STATE"
  fi
}

print_archive_url() {
  local MIGRATION_ID="$1"
  local MIGRATION_STATE="${MIGRATIONS[$MIGRATION_ID]}"

  if [[ $GH_HOST != "" ]]; then
    if [[ "$GH_HOST" == "github.com" ]]; then
      BASE_URL="https://api.github.com"
    else
      BASE_URL="https://$GH_HOST/api/v3"
    fi
  else
    BASE_URL="https://api.github.com"
  fi

  local TOKEN=$(gh auth token)

  if [[ "$MIGRATION_STATE" == "exported" ]]; then
    local ARCHIVE_URL=$(curl -l --no-progress-meter -H "Authorization: bearer $TOKEN" "$BASE_URL/orgs/$ORGANIZATION/migrations/$MIGRATION_ID/archive")
    printf "Archive URL for migration %s: %s\n" "$MIGRATION_ID" "$ARCHIVE_URL"
  else
    printf "Skipping migration %s due to '%s' state; please investigate\n" "$MIGRATION_ID" "$MIGRATION_STATE"
  fi
}

for MIGRATION_ID in "${!MIGRATIONS[@]}"; do
  if $SKIP_DOWNLOAD; then
    print_archive_url "$MIGRATION_ID"
  elif $ARCHIVE_PER_REPO; then
    download_migration "$MIGRATION_ID" "${ARCHIVE:-migration-archive}-$MIGRATION_ID.tar.gz"
  else
    download_migration "$MIGRATION_ID" "${ARCHIVE:-migration-archive-$MIGRATION_ID}.tar.gz"
  fi
done