Skip to content

Commit 42c41f6

Browse files
romangeromange
authored
chore: Finish with GCS::ListBuckets (#292)
1 parent 5c80d30 commit 42c41f6

File tree

3 files changed

+90
-41
lines changed

3 files changed

+90
-41
lines changed

‎examples/gcs_demo.cc

+6-5
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,12 @@ void Run(SSL_CTX* ctx) {
2828
cloud::GCS gcs(&provider, ctx, pb);
2929
ec = gcs.Connect(connect_ms);
3030
CHECK(!ec) << "Could not connect " << ec;
31-
auto res = gcs.ListBuckets();
32-
CHECK(res) << res.error().message();
33-
for (auto v : *res) {
34-
CONSOLE_INFO << v;
35-
}
31+
auto cb = [](std::string_view bname) {
32+
CONSOLE_INFO << bname;
33+
};
34+
35+
ec = gcs.ListBuckets(cb);
36+
CHECK(!ec) << ec.message();
3637
}
3738

3839
int main(int argc, char** argv) {

‎util/cloud/gcp/gcs.cc

+81-34
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,18 @@ auto Unexpected(std::errc code) {
3131
return nonstd::make_unexpected(make_error_code(code));
3232
}
3333

34+
#define RETURN_UNEXPECTED(x) do { \
35+
auto ec = (x); \
36+
if (ec) \
37+
return nonstd::make_unexpected(ec); \
38+
} while(false)
39+
40+
#define RETURN_ERROR(x) do { \
41+
auto ec = (x); \
42+
if (ec) \
43+
return ec; \
44+
} while (false)
45+
3446
string AuthHeader(string_view access_token) {
3547
return absl::StrCat("Bearer ", access_token);
3648
}
@@ -162,35 +174,34 @@ io::Result<TokenTtl> ParseTokenResponse(std::string&& response) {
162174
return result;
163175
}
164176

165-
template <typename RespBody>
166-
error_code SendWithToken(GCPCredsProvider* provider, http::Client* client, EmptyRequest* req, h2::response<RespBody>* resp) {
177+
using EmptyParserPtr = std::unique_ptr<h2::response_parser<h2::empty_body>>;
178+
io::Result<EmptyParserPtr> SendWithToken(GCPCredsProvider* provider, http::Client* client,
179+
EmptyRequest* req) {
180+
error_code ec;
167181
for (unsigned i = 0; i < 2; ++i) { // Iterate for possible token refresh.
168182
VLOG(1) << "HttpReq" << i << ": " << *req << ", socket " << client->native_handle();
169183

170-
error_code ec = client->Send(*req, resp);
171-
if (ec) {
172-
return ec;
173-
}
174-
VLOG(1) << "HttpResp" << i << ": " << *resp;
184+
RETURN_UNEXPECTED(client->Send(*req));
185+
EmptyParserPtr parser(new h2::response_parser<h2::empty_body>());
186+
RETURN_UNEXPECTED(client->ReadHeader(parser.get()));
175187

176-
if (resp->result() == h2::status::ok) {
177-
break;
178-
};
188+
VLOG(1) << "RespHeader" << i << ": " << parser.get();
179189

180-
if (IsUnauthorized(*resp)) {
181-
ec = provider->RefreshToken(client->proactor());
182-
if (ec) {
183-
return ec;
184-
}
190+
if (parser->get().result() == h2::status::ok) {
191+
return parser;
192+
};
185193

186-
*resp = {};
194+
if (IsUnauthorized(parser->get())) {
195+
RETURN_UNEXPECTED(provider->RefreshToken(client->proactor()));
187196
req->set(h2::field::authorization, AuthHeader(provider->access_token()));
188197

189198
continue;
190199
}
191-
LOG(FATAL) << "Unexpected response " << *resp;
200+
ec = make_error_code(errc::bad_message);
201+
LOG(DFATAL) << "Unexpected response " << parser.get();
192202
}
193-
return {};
203+
204+
return nonstd::make_unexpected(ec);
194205
}
195206

196207
} // namespace
@@ -218,18 +229,16 @@ error_code GCPCredsProvider::Init(unsigned connect_ms, fb2::ProactorBase* pb) {
218229
use_instance_metadata_ = true;
219230
LOG(FATAL) << "TBD: do not support reading from instance metadata";
220231
} else {
221-
error_code ec = LoadGCPConfig(&account_id_, &project_id_);
222-
if (ec)
223-
return ec;
232+
RETURN_ERROR(LoadGCPConfig(&account_id_, &project_id_));
233+
224234
if (account_id_.empty() || project_id_.empty()) {
225235
LOG(WARNING) << "gcloud config file is not valid";
226236
return make_error_code(errc::not_supported);
227237
}
228238
string adc_file = absl::StrCat(*root_path, "/legacy_credentials/", account_id_, "/adc.json");
229239
VLOG(1) << "ADC file: " << adc_file;
230-
ec = ParseADC(adc_file, &client_id_, &client_secret_, &refresh_token_);
231-
if (ec)
232-
return ec;
240+
RETURN_ERROR(ParseADC(adc_file, &client_id_, &client_secret_, &refresh_token_));
241+
233242
if (client_id_.empty() || client_secret_.empty() || refresh_token_.empty()) {
234243
LOG(WARNING) << "Bad ADC file " << adc_file;
235244
return make_error_code(errc::bad_message);
@@ -264,9 +273,8 @@ error_code GCPCredsProvider::RefreshToken(fb2::ProactorBase* pb) {
264273
VLOG(1) << "Req: " << req;
265274

266275
h2::response<h2::string_body> resp;
267-
ec = https_client.Send(req, &resp);
268-
if (ec)
269-
return ec;
276+
RETURN_ERROR(https_client.Send(req, &resp));
277+
270278
if (resp.result() != h2::status::ok) {
271279
LOG(WARNING) << "Http error: " << string(resp.reason()) << ", Body: ", resp.body();
272280
return make_error_code(errc::permission_denied);
@@ -297,18 +305,57 @@ std::error_code GCS::Connect(unsigned msec) {
297305
return client_->Connect(kDomain, "443", ssl_ctx_);
298306
}
299307

300-
auto GCS::ListBuckets() -> ListBucketResult {
308+
error_code GCS::ListBuckets(ListBucketCb cb) {
301309
string url = absl::StrCat("/storage/v1/b?project=", creds_provider_.project_id());
302-
absl::StrAppend(&url, "&fields=items,nextPageToken");
310+
absl::StrAppend(&url, "&maxResults=50&fields=items,nextPageToken");
303311

304312
auto http_req = PrepareRequest(h2::verb::get, url, creds_provider_.access_token());
305313

306314
rj::Document doc;
307-
h2::response<h2::string_body> resp_msg;
308-
error_code ec = SendWithToken(&creds_provider_, client_.get(), &http_req, &resp_msg);
309-
if (ec)
310-
return nonstd::make_unexpected(ec);
311-
VLOG(2) << "ListResponse: " << resp_msg.body();
315+
316+
while (true) {
317+
io::Result<EmptyParserPtr> parse_res =
318+
SendWithToken(&creds_provider_, client_.get(), &http_req);
319+
if (!parse_res)
320+
return parse_res.error();
321+
EmptyParserPtr empty_parser = std::move(*parse_res);
322+
h2::response_parser<h2::string_body> resp(std::move(*empty_parser));
323+
RETURN_ERROR(client_->Recv(&resp));
324+
325+
auto msg = resp.release();
326+
327+
VLOG(2) << "ListResponse: " << msg.body();
328+
329+
doc.ParseInsitu(&msg.body().front());
330+
if (doc.HasParseError()) {
331+
return make_error_code(errc::bad_message);
332+
}
333+
334+
auto it = doc.FindMember("items");
335+
if (it == doc.MemberEnd())
336+
break;
337+
338+
const auto& val = it->value;
339+
if (!val.IsArray()) {
340+
return make_error_code(errc::bad_message);
341+
}
342+
auto array = val.GetArray();
343+
344+
for (size_t i = 0; i < array.Size(); ++i) {
345+
const auto& item = array[i];
346+
auto it = item.FindMember("name");
347+
if (it != item.MemberEnd()) {
348+
cb(string_view{it->value.GetString(), it->value.GetStringLength()});
349+
}
350+
}
351+
352+
it = doc.FindMember("nextPageToken");
353+
if (it == doc.MemberEnd()) {
354+
break;
355+
}
356+
absl::string_view page_token{it->value.GetString(), it->value.GetStringLength()};
357+
http_req.target(absl::StrCat(url, "&pageToken=", page_token));
358+
}
312359
return {};
313360
}
314361

‎util/cloud/gcp/gcs.h

+3-2
Original file line numberDiff line numberDiff line change
@@ -69,14 +69,15 @@ class GCPCredsProvider {
6969

7070
class GCS {
7171
public:
72-
using ListBucketResult = io::Result<std::vector<std::string>>;
72+
using BucketItem = std::string_view;
73+
using ListBucketCb = std::function<void(BucketItem)>;
7374

7475
GCS(GCPCredsProvider* creds_provider, SSL_CTX* ssl_cntx, fb2::ProactorBase* pb);
7576
~GCS();
7677

7778
std::error_code Connect(unsigned msec);
7879

79-
ListBucketResult ListBuckets();
80+
std::error_code ListBuckets(ListBucketCb cb);
8081

8182
private:
8283
GCPCredsProvider& creds_provider_;

0 commit comments

Comments
 (0)