From b7e5fad8c17cc2fbdaddf45979cb81ad3d1b4ce5 Mon Sep 17 00:00:00 2001 From: Daniel Azuma Date: Thu, 15 Oct 2020 12:21:50 -0700 Subject: [PATCH] chore: Add synth scripts for client generation (#912) --- .gitignore | 2 + api_list_config.yaml | 8 ++++ bin/generate-api | 92 ++++++++++++++++++++++++++++++++++++-------- script/generate | 8 +--- script/synth.rb | 23 +++++++++++ synth.py | 40 +++++++++++++++++++ 6 files changed, 151 insertions(+), 22 deletions(-) create mode 100755 script/synth.rb create mode 100644 synth.py diff --git a/.gitignore b/.gitignore index 57dc09b82..f9ede3727 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,5 @@ tmp/ *.iml atlassian* +__pycache__/ +/vendor/ diff --git a/api_list_config.yaml b/api_list_config.yaml index bd053b36b..de4986c06 100644 --- a/api_list_config.yaml +++ b/api_list_config.yaml @@ -1 +1,9 @@ +include: + - name: appsmarket + version: v2 + discovery_rest_url: https://content.googleapis.com/discovery/v1/apis/appsmarket/v2/rest + - name: youtubePartner + version: v1 + discovery_rest_url: https://content.googleapis.com/discovery/v1/apis/youtubePartner/v1/rest exclude: [] +pause: [] diff --git a/bin/generate-api b/bin/generate-api index 093d6aa0c..ea55c79e2 100755 --- a/bin/generate-api +++ b/bin/generate-api @@ -28,32 +28,34 @@ module Google desc 'gen OUTDIR', 'Generate ruby API from an API description' method_options url: :array, file: :array, from_discovery: :boolean, preferred_only: :boolean, - verbose: :boolean, names: :string, names_out: :string + verbose: :boolean, names: :string, names_out: :string, api: :string, clean: :boolean def gen(dir) ensure_active_support require 'google/apis/generator' self.destination_root = dir Google::Apis.logger.level = Logger::DEBUG if options[:verbose] - generate_from_url(options[:url]) if options[:url] - generate_from_file(options[:file]) if options[:file] - generate_from_discovery(preferred_only: options[:preferred_only]) if options[:from_discovery] - create_file(options[:names_out]) { |*| generator.dump_api_names } if options[:names_out] + count = 0 + count += generate_from_url(options[:url]) if options[:url] + count += generate_from_file(options[:file]) if options[:file] + count += generate_specific_apis(options[:api]) if options[:api] + count += generate_from_discovery(preferred_only: options[:preferred_only]) if options[:from_discovery] + count += clean_from_discovery if options[:clean] + create_file(options[:names_out]) { |*| generator.dump_api_names } if count > 0 && options[:names_out] end desc 'list', 'List public APIs' method_options verbose: :boolean, preferred_only: :boolean def list Google::Apis.logger.level = Logger::DEBUG if options[:verbose] - discovery = Discovery::DiscoveryService.new - apis = discovery.list_apis - apis.items.each do |api| + discovery_api_list.each do |api| say sprintf('%s - %s', api.id, api.description).strip unless options[:preferred_only] && !api.preferred? end end no_commands do - def generate_from_url(urls) + def generate_from_url(urls, first_only: false) + count = 0 Array(urls).each do |url| begin json = discovery.http(:get, url) @@ -62,7 +64,10 @@ module Google next end generate_api(json) + return 1 if first_only + count += 1 end + count end def generate_from_file(files) @@ -71,15 +76,38 @@ module Google generate_api(f.read) end end + Array(files).size + end + + def generate_specific_apis(apis) + discovery_apis = discovery_api_list.each_with_object({}) do |api, hash| + hash["#{api.name}.#{api.version}"] = api + end + paused_apis = Array(api_list_config["pause"]) + count = 0 + Array(apis).each do |name_version| + api = discovery_apis[name_version] + if api.nil? + say "API #{api} is not in the discovery list." + elsif paused_apis.include? name_version + say "Ignoring paused API #{api.name} #{api.version}" + else + discovery_rest_url = "https://raw.githubusercontent.com/googleapis/discovery-artifact-manager/master/discoveries/#{api.name}.#{api.version}.json" + say sprintf('Loading %s, version %s from %s', api.name, api.version, discovery_rest_url) + generate_from_url([discovery_rest_url, api.discovery_rest_url], first_only: true) + count += 1 + end + end + count end def generate_from_discovery(preferred_only: false) say 'Fetching API list' - apis = discovery.list_apis - exclude_apis = api_list_config["exclude"] || [] - apis.items.each do |api| - if exclude_apis.include? "#{api.name}.#{api.version}" - say "Ignoring excluded API #{api.name} #{api.version}" + paused_apis = Array(api_list_config["pause"]) + count = 0 + discovery_api_list.each do |api| + if paused_apis.include? "#{api.name}.#{api.version}" + say "Ignoring paused API #{api.name} #{api.version}" elsif (preferred_only && !api.preferred?) say sprintf('Ignoring disoverable API %s', api.id) else @@ -91,9 +119,27 @@ module Google # by the Discovery index. discovery_rest_url = "https://raw.githubusercontent.com/googleapis/discovery-artifact-manager/master/discoveries/#{api.name}.#{api.version}.json" say sprintf('Loading %s, version %s from %s', api.name, api.version, discovery_rest_url) - generate_from_url(discovery_rest_url) + generate_from_url([discovery_rest_url, api.discovery_rest_url], first_only: true) + count += 1 end end + count + end + + def clean_from_discovery + count = 0 + apis = discovery_api_list.map { |api| "#{api.name.underscore}_#{api.version.tr '.', '_'}" } + Dir.chdir("#{destination_root}/google/apis") do + Dir.glob("*.rb").each do |filename| + filename = File.basename(filename, ".rb") + unless apis.include? filename + FileUtils.rm_r filename + FileUtils.rm "#{filename}.rb" + count += 1 + end + end + end + count end def generate_api(json) @@ -107,6 +153,20 @@ module Google @discovery ||= Discovery::DiscoveryService.new end + def discovery_api_list + @discovery_api_list ||= begin + items = discovery.list_apis.items + excluded = Array(api_list_config["exclude"]) + items.delete_if { |item| excluded.include? "#{item.name}.#{item.version}" } + Array(api_list_config["include"]).each do |include_hash| + include_hash.symbolize_keys! + include_item = Discovery::DirectoryList::Item.new(**include_hash) + items << include_item unless items.any? { |item| item.name == include_item.name && item.version == include_item.version } + end + items + end + end + def generator @generator ||= Google::Apis::Generator.new(api_names: options[:names], api_names_out: options[:names_out]) end @@ -117,7 +177,9 @@ module Google def ensure_active_support begin + require 'active_support' require 'active_support/inflector' + require 'active_support/core_ext' rescue LoadError => e error 'ActiveSupport is required, please run:' error 'gem install activesupport' diff --git a/script/generate b/script/generate index ab2f46b4e..e64a80d39 100755 --- a/script/generate +++ b/script/generate @@ -4,10 +4,4 @@ DIR=$(dirname $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )) -URLS=(https://content.googleapis.com/discovery/v1/apis/appsmarket/v2/rest \ - https://content.googleapis.com/discovery/v1/apis/youtubePartner/v1/rest \ - https://content.googleapis.com/discovery/v1/apis/compute/beta/rest \ - https://monitoring.googleapis.com/\$discovery/rest?version=v3 -) - -echo 'a' | bundle exec bin/generate-api gen generated --from-discovery --no-preferred-only --names-out=$DIR/api_names_out.yaml --url=${URLS[*]} +echo 'a' | bundle exec bin/generate-api gen generated --from-discovery --no-preferred-only --names-out=$DIR/api_names_out.yaml diff --git a/script/synth.rb b/script/synth.rb new file mode 100755 index 000000000..a7854dda0 --- /dev/null +++ b/script/synth.rb @@ -0,0 +1,23 @@ +#!/usr/bin/env ruby + +require "fileutils" + +DIR = File.dirname __dir__ + +def execute cmd + puts cmd + abort unless system cmd +end + +execute "bundle install" + +if ARGV.empty? + execute "echo a | bundle exec bin/generate-api gen generated --from-discovery --no-preferred-only --names-out=#{DIR}/api_names_out.yaml" +elsif ARGV == ["--clean"] + execute "bundle exec bin/generate-api gen generated --clean" +elsif ARGV.size == 2 + api, version = ARGV + execute "echo a | bundle exec bin/generate-api gen generated --api=#{api}.#{version} --names-out=#{DIR}/api_names_out.yaml" +else + abort "Bad arguments: #{ARGV}" +end diff --git a/synth.py b/synth.py new file mode 100644 index 000000000..67bb9953c --- /dev/null +++ b/synth.py @@ -0,0 +1,40 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This script is used to synthesize generated parts of this library.""" + +from synthtool.__main__ import extra_args +import synthtool as s +import synthtool.log as log +import synthtool.shell as shell +import logging +import os + +logging.basicConfig(level=logging.DEBUG) +s.metadata.set_track_obsolete_files(False) # TODO: enable again. + +command = [ + "docker", + "run", + "--rm", + f"-v{os.getcwd()}:/workspace", + "-v/var/run/docker.sock:/var/run/docker.sock", + "-w", "/workspace", + "--entrypoint", "script/synth.rb", + "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/autosynth"] +if extra_args(): + command.extend(extra_args()) + +log.debug(f"Running: {' '.join(command)}") +shell.run(command, hide_output=False)