Skip to content

Commit 9a2f966

Browse files
amanda-tarafajskeet
amanda-tarafa
authored and
jskeet
committed
Adds BigQueryInsertResults which wraps the response obtained from insert attempts.
1 parent 61c9e79 commit 9a2f966

File tree

5 files changed

+855
-0
lines changed

5 files changed

+855
-0
lines changed

‎apis/Google.Cloud.BigQuery.V2/Google.Cloud.BigQuery.V2.Tests/BigQueryInsertResultsTests.cs‎

Lines changed: 352 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,352 @@
1+
// Copyright 2020 Google Inc. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
using Google.Apis.Bigquery.v2;
16+
using Google.Apis.Bigquery.v2.Data;
17+
using Google.Apis.Requests;
18+
using System;
19+
using System.Collections.Generic;
20+
using System.Linq;
21+
using Xunit;
22+
using static Google.Apis.Bigquery.v2.Data.TableDataInsertAllResponse;
23+
24+
namespace Google.Cloud.BigQuery.V2.Tests
25+
{
26+
public class BigQueryInsertResultsTests
27+
{
28+
private readonly BigQueryClient _client;
29+
30+
public BigQueryInsertResultsTests()
31+
{
32+
_client = new DerivedBigQueryClient();
33+
}
34+
35+
public static IEnumerable<object[]> NotNullableConstructorParametersData
36+
{
37+
get
38+
{
39+
yield return new object[] { null, new InsertOptions(), new List<BigQueryInsertRow>().AsReadOnly(), new TableDataInsertAllResponse() };
40+
yield return new object[] { new DerivedBigQueryClient(), new InsertOptions(), null, new TableDataInsertAllResponse() };
41+
yield return new object[] { new DerivedBigQueryClient(), new InsertOptions(), new List<BigQueryInsertRow>().AsReadOnly(), null };
42+
}
43+
}
44+
45+
[Theory]
46+
[MemberData(nameof(NotNullableConstructorParametersData))]
47+
public void Constructor_NullParameters_Fail(
48+
BigQueryClient client, InsertOptions options, IReadOnlyList<BigQueryInsertRow> rows, TableDataInsertAllResponse response) =>
49+
Assert.Throws<ArgumentNullException>(() => new BigQueryInsertResults(client, options, rows, response));
50+
51+
[Fact]
52+
public void Constructor_NullOptions() =>
53+
Assert.NotNull(new BigQueryInsertResults(_client, null, new List<BigQueryInsertRow>(), new TableDataInsertAllResponse()));
54+
55+
public static IEnumerable<object[]> NoErrorsData
56+
{
57+
get
58+
{
59+
yield return new object[] { new TableDataInsertAllResponse() };
60+
yield return new object[] { new TableDataInsertAllResponse { InsertErrors = new List<InsertErrorsData>() } };
61+
yield return new object[] { new TableDataInsertAllResponse { InsertErrors = new List<InsertErrorsData> { null, null, null } } };
62+
}
63+
}
64+
65+
[Theory]
66+
[MemberData(nameof(NoErrorsData))]
67+
public void AllRowsInserted(TableDataInsertAllResponse response)
68+
{
69+
BigQueryInsertResults results = new BigQueryInsertResults(
70+
_client,
71+
new InsertOptions(),
72+
new List<BigQueryInsertRow> { new BigQueryInsertRow() },
73+
response);
74+
75+
Assert.Equal(BigQueryInsertStatus.AllRowsInserted, results.Status);
76+
Assert.Equal(0, results.OriginalRowsWithErrors);
77+
Assert.Equal(1, results.InsertAttemptRowCount);
78+
Assert.Empty(results.Errors);
79+
Assert.Same(results, results.ThrowOnNoneInserted());
80+
Assert.Same(results, results.ThrowOnNotAllInserted());
81+
Assert.Same(results, results.ThrowOnAnyError());
82+
}
83+
84+
[Fact]
85+
public void SomeRowsInserted()
86+
{
87+
ErrorProto row1Error1 = new ErrorProto { Location = "field_1", Reason = "reason_1", Message = "message_1" };
88+
ErrorProto row1Error2 = new ErrorProto { Location = "field_2", Message = "message_2" };
89+
InsertErrorsData row1List1 = new InsertErrorsData { Index = 1, Errors = new List<ErrorProto> { row1Error1, row1Error2 } };
90+
91+
ErrorProto row5Error1 = new ErrorProto { Location = "field_3", Reason = "reason_3" };
92+
InsertErrorsData row5List1 = new InsertErrorsData { Index = 5, Errors = new List<ErrorProto> { row5Error1 } };
93+
94+
InsertErrorsData row6List1 = new InsertErrorsData { Index = 6 };
95+
96+
ErrorProto row1Error3 = new ErrorProto { Location = "field_4", Reason = "reason_4", Message = "message_4" };
97+
InsertErrorsData row1List2 = new InsertErrorsData { Index = 1, Errors = new List<ErrorProto> { row1Error3 } };
98+
99+
TableDataInsertAllResponse response = new TableDataInsertAllResponse
100+
{
101+
InsertErrors = new List<InsertErrorsData>
102+
{
103+
row1List1,
104+
row5List1,
105+
row6List1,
106+
row1List2
107+
}
108+
};
109+
110+
IReadOnlyList<BigQueryInsertRow> rows = Enumerable.Range(0, 8).Select(_ => new BigQueryInsertRow()).ToList().AsReadOnly();
111+
112+
List<BigQueryInsertRowErrors> expectedInsertRowErrors = new List<BigQueryInsertRowErrors>
113+
{
114+
new BigQueryInsertRowErrors(rows[1], new List<InsertErrorsData> { row1List1, row1List2 }),
115+
new BigQueryInsertRowErrors(rows[5], new List<InsertErrorsData> { row5List1 }),
116+
new BigQueryInsertRowErrors(rows[6], new List<InsertErrorsData> { row6List1 }),
117+
};
118+
List<SingleError> expectedSingleErrors = expectedInsertRowErrors.SelectMany(rowError => rowError).ToList();
119+
120+
BigQueryInsertResults results = new BigQueryInsertResults(
121+
_client,
122+
new InsertOptions { SkipInvalidRows = true },
123+
rows,
124+
response);
125+
126+
Assert.Equal(BigQueryInsertStatus.SomeRowsInserted, results.Status);
127+
Assert.Equal(3, results.OriginalRowsWithErrors);
128+
Assert.Equal(8, results.InsertAttemptRowCount);
129+
Assert.Equal(expectedInsertRowErrors, results.Errors.ToList(), new BigQueryInsertRowErrorEqualityComparer());
130+
Assert.Same(results, results.ThrowOnNoneInserted());
131+
AssertException(results.ThrowOnNotAllInserted, expectedSingleErrors, BigQueryInsertStatus.SomeRowsInserted);
132+
AssertException(results.ThrowOnAnyError, expectedSingleErrors, BigQueryInsertStatus.SomeRowsInserted);
133+
}
134+
135+
[Fact]
136+
public void NoRowsInserted_NoSkip()
137+
{
138+
ErrorProto row1Error1 = new ErrorProto { Location = "field_1", Reason = "reason_1", Message = "message_1" };
139+
ErrorProto row1Error2 = new ErrorProto { Location = "field_2", Message = "message_2" };
140+
InsertErrorsData row1List1 = new InsertErrorsData { Index = 1, Errors = new List<ErrorProto> { row1Error1, row1Error2 } };
141+
142+
ErrorProto row5Error1 = new ErrorProto { Location = "field_3", Reason = "reason_3" };
143+
InsertErrorsData row5List1 = new InsertErrorsData { Index = 5, Errors = new List<ErrorProto> { row5Error1 } };
144+
145+
InsertErrorsData row6List1 = new InsertErrorsData { Index = 6 };
146+
147+
ErrorProto row1Error3 = new ErrorProto { Location = "field_4", Reason = "reason_4", Message = "message_4" };
148+
InsertErrorsData row1List2 = new InsertErrorsData { Index = 1, Errors = new List<ErrorProto> { row1Error3 } };
149+
150+
TableDataInsertAllResponse response = new TableDataInsertAllResponse
151+
{
152+
InsertErrors = new List<InsertErrorsData>
153+
{
154+
row1List1,
155+
row5List1,
156+
row6List1,
157+
row1List2
158+
}
159+
};
160+
161+
IReadOnlyList<BigQueryInsertRow> rows = Enumerable.Range(0, 8).Select(_ => new BigQueryInsertRow()).ToList().AsReadOnly();
162+
163+
// We don't need to have errors for all rows to know that no rows were inserted.
164+
// If SkipInvalidRows is not set to true, then even if only one row has errors, no rows are inserted.
165+
List<BigQueryInsertRowErrors> expectedInsertRowErrors = new List<BigQueryInsertRowErrors>
166+
{
167+
new BigQueryInsertRowErrors(rows[1], new List<InsertErrorsData> { row1List1, row1List2 }),
168+
new BigQueryInsertRowErrors(rows[5], new List<InsertErrorsData> { row5List1 }),
169+
new BigQueryInsertRowErrors(rows[6], new List<InsertErrorsData> { row6List1 }),
170+
};
171+
List<SingleError> expectedSingleErrors = expectedInsertRowErrors.SelectMany(rowError => rowError).ToList();
172+
173+
BigQueryInsertResults results = new BigQueryInsertResults(
174+
_client,
175+
new InsertOptions(),
176+
rows,
177+
response);
178+
179+
Assert.Equal(BigQueryInsertStatus.NoRowsInserted, results.Status);
180+
Assert.Equal(3, results.OriginalRowsWithErrors);
181+
Assert.Equal(8, results.InsertAttemptRowCount);
182+
Assert.Equal(expectedInsertRowErrors, results.Errors.ToList(), new BigQueryInsertRowErrorEqualityComparer());
183+
AssertException(results.ThrowOnNoneInserted, expectedSingleErrors, BigQueryInsertStatus.NoRowsInserted);
184+
AssertException(results.ThrowOnNotAllInserted, expectedSingleErrors, BigQueryInsertStatus.NoRowsInserted);
185+
AssertException(results.ThrowOnAnyError, expectedSingleErrors, BigQueryInsertStatus.NoRowsInserted);
186+
}
187+
188+
[Fact]
189+
public void NoRowsInserted_AllInvalid()
190+
{
191+
ErrorProto row0Error1 = new ErrorProto { Location = "field_1", Reason = "reason_1", Message = "message_1" };
192+
ErrorProto row0Error2 = new ErrorProto { Location = "field_2", Message = "message_2" };
193+
InsertErrorsData row0List1 = new InsertErrorsData { Index = 0, Errors = new List<ErrorProto> { row0Error1, row0Error2 } };
194+
195+
ErrorProto row1Error1 = new ErrorProto { Location = "field_3", Reason = "reason_3" };
196+
InsertErrorsData row1List1 = new InsertErrorsData { Index = 1, Errors = new List<ErrorProto> { row1Error1 } };
197+
198+
InsertErrorsData row2List1 = new InsertErrorsData { Index = 2 };
199+
200+
ErrorProto row0Error3 = new ErrorProto { Location = "field_4", Reason = "reason_4", Message = "message_4" };
201+
InsertErrorsData row0List2 = new InsertErrorsData { Index = 0, Errors = new List<ErrorProto> { row0Error3 } };
202+
203+
TableDataInsertAllResponse response = new TableDataInsertAllResponse
204+
{
205+
InsertErrors = new List<InsertErrorsData>
206+
{
207+
row0List1,
208+
row1List1,
209+
row2List1,
210+
row0List2
211+
}
212+
};
213+
214+
IReadOnlyList<BigQueryInsertRow> rows = Enumerable.Range(0, 3).Select(_ => new BigQueryInsertRow()).ToList().AsReadOnly();
215+
216+
List<BigQueryInsertRowErrors> expectedInsertRowErrors = new List<BigQueryInsertRowErrors>
217+
{
218+
new BigQueryInsertRowErrors(rows[0], new List<InsertErrorsData> { row0List1, row0List2 }),
219+
new BigQueryInsertRowErrors(rows[1], new List<InsertErrorsData> { row1List1 }),
220+
new BigQueryInsertRowErrors(rows[2], new List<InsertErrorsData> { row2List1 }),
221+
};
222+
List<SingleError> expectedSingleErrors = expectedInsertRowErrors.SelectMany(rowError => rowError).ToList();
223+
224+
BigQueryInsertResults results = new BigQueryInsertResults(
225+
_client,
226+
new InsertOptions { SkipInvalidRows = true },
227+
rows,
228+
response);
229+
230+
Assert.Equal(BigQueryInsertStatus.NoRowsInserted, results.Status);
231+
Assert.Equal(3, results.OriginalRowsWithErrors);
232+
Assert.Equal(3, results.InsertAttemptRowCount);
233+
Assert.Equal(expectedInsertRowErrors, results.Errors.ToList(), new BigQueryInsertRowErrorEqualityComparer());
234+
AssertException(results.ThrowOnNoneInserted, expectedSingleErrors, BigQueryInsertStatus.NoRowsInserted);
235+
AssertException(results.ThrowOnNotAllInserted, expectedSingleErrors, BigQueryInsertStatus.NoRowsInserted);
236+
AssertException(results.ThrowOnAnyError, expectedSingleErrors, BigQueryInsertStatus.NoRowsInserted);
237+
}
238+
239+
[Fact]
240+
public void UnexpectedResponse()
241+
{
242+
ErrorProto row1Error1 = new ErrorProto { Location = "field_1", Reason = "reason_1", Message = "message_1" };
243+
ErrorProto row1Error2 = new ErrorProto { Location = "field_2", Message = "message_2" };
244+
InsertErrorsData row1List1 = new InsertErrorsData { Index = 1, Errors = new List<ErrorProto> { row1Error1, row1Error2 } };
245+
246+
ErrorProto row5Error1 = new ErrorProto { Location = "field_3", Reason = "reason_3" };
247+
InsertErrorsData row5List1 = new InsertErrorsData { Index = 5, Errors = new List<ErrorProto> { row5Error1 } };
248+
249+
InsertErrorsData row6List1 = new InsertErrorsData { Index = 6 };
250+
251+
ErrorProto row1Error3 = new ErrorProto { Location = "field_4", Reason = "reason_4", Message = "message_4" };
252+
InsertErrorsData row1List2 = new InsertErrorsData { Index = 1, Errors = new List<ErrorProto> { row1Error3 } };
253+
254+
// Row 11, which we don't have
255+
ErrorProto row11Error1 = new ErrorProto { Location = "field_5", Reason = "reason_5", Message = "message_5" };
256+
InsertErrorsData row11List1 = new InsertErrorsData { Index = 11, Errors = new List<ErrorProto> { row11Error1 } };
257+
258+
// No row index specified
259+
ErrorProto rowNullError1 = new ErrorProto { Location = "field_6", Reason = "reason_6", Message = "message_6" };
260+
InsertErrorsData rowNullList1 = new InsertErrorsData { Index = null, Errors = new List<ErrorProto> { rowNullError1 } };
261+
262+
TableDataInsertAllResponse response = new TableDataInsertAllResponse
263+
{
264+
InsertErrors = new List<InsertErrorsData>
265+
{
266+
row1List1,
267+
row5List1,
268+
null, // Ignored
269+
row6List1,
270+
row11List1,
271+
row1List2,
272+
rowNullList1
273+
}
274+
};
275+
276+
IReadOnlyList<BigQueryInsertRow> rows = Enumerable.Range(0, 8).Select(_ => new BigQueryInsertRow()).ToList().AsReadOnly();
277+
278+
List<BigQueryInsertRowErrors> expectedInsertRowErrors = new List<BigQueryInsertRowErrors>
279+
{
280+
new BigQueryInsertRowErrors(rows[1], new List<InsertErrorsData> { row1List1, row1List2 }),
281+
new BigQueryInsertRowErrors(rows[5], new List<InsertErrorsData> { row5List1 }),
282+
new BigQueryInsertRowErrors(rows[6], new List<InsertErrorsData> { row6List1 }),
283+
new BigQueryInsertRowErrors(null, new List<InsertErrorsData> { row11List1 }),
284+
new BigQueryInsertRowErrors(null, new List<InsertErrorsData> { rowNullList1 }),
285+
};
286+
List<SingleError> expectedSingleErrors = expectedInsertRowErrors.SelectMany(rowError => rowError).ToList();
287+
288+
BigQueryInsertResults results = new BigQueryInsertResults(
289+
_client,
290+
new InsertOptions { SkipInvalidRows = true },
291+
rows,
292+
response);
293+
294+
Assert.Equal(BigQueryInsertStatus.SomeRowsInserted, results.Status);
295+
Assert.Equal(3, results.OriginalRowsWithErrors);
296+
Assert.Equal(8, results.InsertAttemptRowCount);
297+
Assert.Equal(expectedInsertRowErrors, results.Errors.ToList(), new BigQueryInsertRowErrorEqualityComparer());
298+
Assert.Same(results, results.ThrowOnNoneInserted());
299+
AssertException(results.ThrowOnNotAllInserted, expectedSingleErrors, BigQueryInsertStatus.SomeRowsInserted);
300+
AssertException(results.ThrowOnAnyError, expectedSingleErrors, BigQueryInsertStatus.SomeRowsInserted);
301+
}
302+
303+
private void AssertException(Func<BigQueryInsertResults> throwing, List<SingleError> expectedErrors, BigQueryInsertStatus expectedStatus)
304+
{
305+
GoogleApiException exception = Assert.Throws<GoogleApiException>(throwing);
306+
Assert.Equal("fake_service_name", exception.ServiceName);
307+
Assert.Contains(expectedStatus.ToString(), exception.Message);
308+
Assert.Equal(expectedErrors, exception.Error.Errors, new SingleErrorEqualityComparer());
309+
}
310+
311+
public class DerivedBigQueryClient : BigQueryClient
312+
{
313+
private readonly BigqueryService _service = new DerivedBigQueryService();
314+
public override BigqueryService Service => _service;
315+
}
316+
317+
public class DerivedBigQueryService : BigqueryService
318+
{
319+
public override string Name => "fake_service_name";
320+
}
321+
322+
public class BigQueryInsertRowErrorEqualityComparer : IEqualityComparer<BigQueryInsertRowErrors>
323+
{
324+
public bool Equals(BigQueryInsertRowErrors x, BigQueryInsertRowErrors y)
325+
{
326+
if (x == y)
327+
{
328+
return true;
329+
}
330+
if (x == null || y == null)
331+
{
332+
return false;
333+
}
334+
335+
return x.OriginalRowIndex == y.OriginalRowIndex &&
336+
// This is good enough for the purposes of this test.
337+
x.OriginalRow == y.OriginalRow &&
338+
x.SequenceEqual(y, new SingleErrorEqualityComparer());
339+
}
340+
341+
public int GetHashCode(BigQueryInsertRowErrors obj) => throw new NotImplementedException();
342+
}
343+
344+
public class SingleErrorEqualityComparer : IEqualityComparer<SingleError>
345+
{
346+
public bool Equals(SingleError x, SingleError y) =>
347+
string.Equals(x?.ToString(), y?.ToString(), StringComparison.InvariantCulture);
348+
349+
public int GetHashCode(SingleError obj) => throw new NotImplementedException();
350+
}
351+
}
352+
}

0 commit comments

Comments
 (0)