|
1 | 1 | /*
|
2 |
| - Copyright 2015, Google, Inc. |
3 |
| - Licensed under the Apache License, Version 2.0 (the "License"); |
4 |
| - you may not use this file except in compliance with the License. |
5 |
| - You may obtain a copy of the License at |
6 |
| - |
7 |
| - http://www.apache.org/licenses/LICENSE-2.0 |
8 |
| - |
9 |
| - Unless required by applicable law or agreed to in writing, software |
10 |
| - distributed under the License is distributed on an "AS IS" BASIS, |
11 |
| - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 |
| - See the License for the specific language governing permissions and |
| 2 | + Copyright 2015, Google, Inc. |
| 3 | + Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | + you may not use this file except in compliance with the License. |
| 5 | + You may obtain a copy of the License at |
| 6 | +
|
| 7 | + http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +
|
| 9 | + Unless required by applicable law or agreed to in writing, software |
| 10 | + distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | + See the License for the specific language governing permissions and |
13 | 13 | limitations under the License.
|
14 | 14 | */
|
15 | 15 | package com.google.cloud.bigquery.samples;
|
|
29 | 29 | import java.util.Scanner;
|
30 | 30 |
|
31 | 31 | /**
|
32 |
| - * TODO: Insert description here. (generated by elibixby) |
| 32 | + * Cli tool to load data from a CSV into Bigquery. |
33 | 33 | */
|
34 |
| -public class LoadDataCSVSample extends BigqueryUtils { |
| 34 | +public class LoadDataCSVSample { |
| 35 | + |
| 36 | + /** |
| 37 | + * Protected constructor since this is a collection of static methods. |
| 38 | + */ |
| 39 | + protected LoadDataCSVSample() { |
35 | 40 |
|
36 |
| - |
| 41 | + } |
| 42 | + |
| 43 | + /** |
| 44 | + * Cli tool to load data from a CSV into Bigquery. |
| 45 | + * @param args Command line args, should be empty |
| 46 | + * @throws IOException IOException |
| 47 | + * @throws InterruptedException InterruptedException |
| 48 | + */ |
37 | 49 | // [START main]
|
38 |
| - public static void main(String[] args) throws IOException, InterruptedException{ |
| 50 | + public static void main(final String[] args) throws IOException, |
| 51 | + InterruptedException { |
39 | 52 | Scanner scanner = new Scanner(System.in);
|
40 | 53 | System.out.println("Enter your project id: ");
|
41 | 54 | String projectId = scanner.nextLine();
|
42 | 55 | System.out.println("Enter your dataset id: ");
|
43 | 56 | String datasetId = scanner.nextLine();
|
44 | 57 | System.out.println("Enter your table id: ");
|
45 | 58 | String tableId = scanner.nextLine();
|
46 |
| - System.out.println("Enter the Google Cloud Storage Path to the data you'd like to load: "); |
| 59 | + System.out.println("Enter the Google Cloud Storage Path to the data " |
| 60 | + + "you'd like to load: "); |
47 | 61 | String cloudStoragePath = scanner.nextLine();
|
48 | 62 | System.out.println("Enter the filepath to your schema: ");
|
49 | 63 | String sourceSchemaPath = scanner.nextLine();
|
50 |
| - |
51 |
| - |
52 |
| - System.out.println("Enter how often to check if your job is complete (milliseconds): "); |
| 64 | + |
| 65 | + |
| 66 | + System.out.println("Enter how often to check if your job is complete " |
| 67 | + + "(milliseconds): "); |
53 | 68 | long interval = scanner.nextLong();
|
54 | 69 | scanner.close();
|
55 |
| - |
| 70 | + |
56 | 71 | run(cloudStoragePath,
|
57 | 72 | projectId,
|
58 | 73 | datasetId,
|
59 | 74 | tableId,
|
60 | 75 | new FileReader(new File(sourceSchemaPath)),
|
61 | 76 | interval);
|
62 |
| - |
| 77 | + |
63 | 78 | }
|
64 | 79 | // [END main]
|
65 |
| - |
| 80 | + |
| 81 | + /** |
| 82 | + * Run the bigquery ClI. |
| 83 | + * @param cloudStoragePath The bucket we are using |
| 84 | + * @param projectId Project id |
| 85 | + * @param datasetId datasetid |
| 86 | + * @param tableId tableid |
| 87 | + * @param schemaSource Source of the schema |
| 88 | + * @param interval interval to wait between polling in milliseconds |
| 89 | + * @throws IOException Thrown if there is an error connecting to Bigquery. |
| 90 | + * @throws InterruptedException Should never be thrown |
| 91 | + */ |
66 | 92 | // [START run]
|
67 |
| - public static void run( |
68 |
| - String cloudStoragePath, |
69 |
| - String projectId, |
70 |
| - String datasetId, |
71 |
| - String tableId, |
72 |
| - Reader schemaSource, |
73 |
| - long interval) throws IOException, InterruptedException{ |
| 93 | + public static void run( |
| 94 | + final String cloudStoragePath, |
| 95 | + final String projectId, |
| 96 | + final String datasetId, |
| 97 | + final String tableId, |
| 98 | + final Reader schemaSource, |
| 99 | + final long interval) throws IOException, InterruptedException { |
74 | 100 |
|
75 | 101 | Bigquery bigquery = BigqueryServiceFactory.getService();
|
76 |
| - |
77 |
| - |
| 102 | + |
| 103 | + |
78 | 104 | Job loadJob = loadJob(
|
79 | 105 | bigquery,
|
80 | 106 | cloudStoragePath,
|
81 | 107 | new TableReference()
|
82 | 108 | .setProjectId(projectId)
|
83 | 109 | .setDatasetId(datasetId)
|
84 | 110 | .setTableId(tableId),
|
85 |
| - loadSchema(schemaSource)); |
| 111 | + BigqueryUtils.loadSchema(schemaSource)); |
86 | 112 |
|
87 |
| - Bigquery.Jobs.Get get_job = bigquery.jobs().get( |
88 |
| - loadJob.getJobReference().getProjectId(), |
| 113 | + Bigquery.Jobs.Get getJob = bigquery.jobs().get( |
| 114 | + loadJob.getJobReference().getProjectId(), |
89 | 115 | loadJob.getJobReference().getJobId());
|
90 |
| - |
91 |
| - pollJob(get_job, interval); |
92 |
| - |
| 116 | + |
| 117 | + BigqueryUtils.pollJob(getJob, interval); |
| 118 | + |
93 | 119 | System.out.println("Load is Done!");
|
94 |
| - |
| 120 | + |
95 | 121 | }
|
96 | 122 | // [END run]
|
97 |
| - |
| 123 | + |
| 124 | + /** |
| 125 | + * A job that extracts data from a table. |
| 126 | + * @param bigquery Bigquery service to use |
| 127 | + * @param cloudStoragePath Cloud storage bucket we are inserting into |
| 128 | + * @param table Table to extract from |
| 129 | + * @param schema The schema of the table we are loading into |
| 130 | + * @return The job to extract data from the table |
| 131 | + * @throws IOException Thrown if error connceting to Bigtable |
| 132 | + */ |
98 | 133 | // [START load_job]
|
99 | 134 | public static Job loadJob(
|
100 |
| - Bigquery bigquery, |
101 |
| - String cloudStoragePath, |
102 |
| - TableReference table, |
103 |
| - TableSchema schema) throws IOException{ |
104 |
| - |
| 135 | + final Bigquery bigquery, |
| 136 | + final String cloudStoragePath, |
| 137 | + final TableReference table, |
| 138 | + final TableSchema schema) throws IOException { |
| 139 | + |
105 | 140 | JobConfigurationLoad load = new JobConfigurationLoad()
|
106 | 141 | .setDestinationTable(table)
|
107 | 142 | .setSchema(schema)
|
108 | 143 | .setSourceUris(Collections.singletonList(cloudStoragePath));
|
109 | 144 |
|
110 |
| - return bigquery.jobs().insert(table.getProjectId(), |
| 145 | + return bigquery.jobs().insert(table.getProjectId(), |
111 | 146 | new Job().setConfiguration(new JobConfiguration().setLoad(load)))
|
112 | 147 | .execute();
|
113 | 148 | }
|
114 | 149 | // [END load_job]
|
115 |
| - |
| 150 | + |
116 | 151 |
|
117 | 152 | }
|
0 commit comments