diff options
| -rw-r--r-- | Android.bp | 2 | ||||
| -rw-r--r-- | fcp/client/fcp_runner.cc | 40 | ||||
| -rw-r--r-- | fcp/client/fcp_runner.h | 3 | ||||
| -rw-r--r-- | fcp/client/fl_runner.proto | 35 | ||||
| -rw-r--r-- | fcp/protos/ondevicepersonalization/eligibility_spec.proto | 61 | ||||
| -rw-r--r-- | fcp/protos/ondevicepersonalization/task_assignments.proto | 20 | ||||
| -rw-r--r-- | fcp/tensorflow/file_descriptor_filesystem.cc | 16 | ||||
| -rw-r--r-- | fcp/tensorflow/file_descriptor_filesystem.h | 16 | ||||
| -rw-r--r-- | fcp/tensorflow/file_descriptor_filesystem_test.cc | 17 |
9 files changed, 203 insertions, 7 deletions
@@ -48,6 +48,7 @@ java_library_static { }, srcs: [ "fcp/protos/ondevicepersonalization/task_assignments.proto", + "fcp/protos/ondevicepersonalization/eligibility_spec.proto", "fcp/protos/federatedcompute/common.proto", "fcp/protos/plan.proto", "fcp/client/**/*.proto", @@ -188,7 +189,6 @@ cc_test { "libprotobuf-cpp-lite-ndk", "liblog", "tensorflow_abseil", - "libc++fs", // used by filesystem ], whole_static_libs: [ "libfederatedcompute", diff --git a/fcp/client/fcp_runner.cc b/fcp/client/fcp_runner.cc index 51e1618..9aba277 100644 --- a/fcp/client/fcp_runner.cc +++ b/fcp/client/fcp_runner.cc @@ -1,3 +1,19 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include "fcp/client/fcp_runner.h" #include "fcp/client/engine/example_iterator_factory.h" @@ -279,11 +295,35 @@ absl::StatusOr<FLRunnerResult> RunFederatedComputation( if (outcome == engine::PlanOutcome::kSuccess) { fl_runner_result.set_contribution_result(FLRunnerResult::SUCCESS); } else { + switch (outcome) { + case engine::PlanOutcome::kInvalidArgument: + fl_runner_result.set_error_status(FLRunnerResult::INVALID_ARGUMENT); + break; + case engine::PlanOutcome::kTensorflowError: + fl_runner_result.set_error_status(FLRunnerResult::TENSORFLOW_ERROR); + break; + case engine::PlanOutcome::kExampleIteratorError: + fl_runner_result.set_error_status( + FLRunnerResult::EXAMPLE_ITERATOR_ERROR); + break; + default: + break; + } fl_runner_result.set_contribution_result(FLRunnerResult::FAIL); std::string error_message = std::string{ plan_result_and_checkpoint_file.plan_result.original_status.message()}; fl_runner_result.set_error_message(error_message); } + + FLRunnerResult::ExampleStats example_stats; + example_stats.set_example_count( + plan_result_and_checkpoint_file.plan_result.example_stats.example_count); + example_stats.set_example_size_bytes( + plan_result_and_checkpoint_file.plan_result.example_stats + .example_size_bytes); + + *fl_runner_result.mutable_example_stats() = example_stats; + return fl_runner_result; } diff --git a/fcp/client/fcp_runner.h b/fcp/client/fcp_runner.h index 0e6d369..6eac8c5 100644 --- a/fcp/client/fcp_runner.h +++ b/fcp/client/fcp_runner.h @@ -1,5 +1,5 @@ /* - * Copyright 2020 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + #ifndef FCP_CLIENT_FCP_RUNNER_H_ #define FCP_CLIENT_FCP_RUNNER_H_ diff --git a/fcp/client/fl_runner.proto b/fcp/client/fl_runner.proto index 55d22f5..863d7f3 100644 --- a/fcp/client/fl_runner.proto +++ b/fcp/client/fl_runner.proto @@ -45,11 +45,42 @@ message FLRunnerResult { } ContributionResult contribution_result = 5; + reserved 2, 3; + + // All fields below are added by OnDevicePersonalization module for logging and + // debugging purpose. // Debug message will be present if ContributionResult is FAIL. - string error_message = 6; + string error_message = 200; - reserved 2, 3; + enum ErrorStatus { + ERROR_STATUS_UNSPECIFIED = 0; + // A TensorFlow error occurred. + TENSORFLOW_ERROR = 1; + // The input parameters are invalid. + INVALID_ARGUMENT = 2; + // An example iterator error occurred. + EXAMPLE_ITERATOR_ERROR = 3; + // Not eligible to execute task. + NOT_ELIGIBLE = 4; + } + + // Used to populate error happens in c++ to java code. Mainly for debug and + // logging purpose. + ErrorStatus error_status = 201; + + message ExampleStats { + // For TensorlowSpec-based plans, this refers to the overall number of + // elements returned by all ExampleIterator::Next() calls. For + // ExampleQuerySpec-based plans, this refers to the total number of row counts + // calculated at example store layer and passed via ExampleQueryResults. + int32 example_count = 1; + int64 example_size_bytes = 2; + } + + // Used to record the stats of examples used in computation. Mainly used for + // debug and logging purpose. + ExampleStats example_stats = 202; } // A suggestion to the client when to retry the connection to the service next diff --git a/fcp/protos/ondevicepersonalization/eligibility_spec.proto b/fcp/protos/ondevicepersonalization/eligibility_spec.proto new file mode 100644 index 0000000..73862d9 --- /dev/null +++ b/fcp/protos/ondevicepersonalization/eligibility_spec.proto @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package google.ondevicepersonalization.federatedcompute.proto; + +option java_package = "com.google.ondevicepersonalization.federatedcompute.proto"; +option java_multiple_files = true; + + +// Provides the information needed to determine eligibility for a task. +// Next Id: 2 +message EligibilityTaskInfo { + // The eligibility policies that apply to this task. + repeated EligibilityPolicyEvalSpec eligibility_policies = 1; +} + +// Specification describing the eligibility policy and its parameters. +// Next Id: 4 +message EligibilityPolicyEvalSpec { + // The identifier of the policy. It should be unique within population. + string id = 1; + + // The specification of the policy implementation, including the + // policy-specific parameters. + oneof policy_type { + MinimumSeparationPolicy min_sep_policy = 2; + DataAvailabilityPolicy data_availability_policy = 3; + } +} + +// Minimum separation policy parameters. +// Next Id: 3 +message MinimumSeparationPolicy { + // The current index (e.g., algorithmic round number) of the federated computation. + int64 current_index = 1; + + // The minimum index separation required between successful contributions. + int64 minimum_separation = 2; +} + +// Data availability policy parameters. +message DataAvailabilityPolicy { + // The minimum number of examples from the selector to be considered + // eligible. + int32 min_example_count = 1; +} diff --git a/fcp/protos/ondevicepersonalization/task_assignments.proto b/fcp/protos/ondevicepersonalization/task_assignments.proto index b890695..c54a2ca 100644 --- a/fcp/protos/ondevicepersonalization/task_assignments.proto +++ b/fcp/protos/ondevicepersonalization/task_assignments.proto @@ -18,6 +18,7 @@ syntax = "proto3"; package google.ondevicepersonalization.federatedcompute.proto; import "fcp/protos/federatedcompute/common.proto"; +import "fcp/protos/ondevicepersonalization/eligibility_spec.proto"; option java_package = "com.google.ondevicepersonalization.federatedcompute.proto"; option java_multiple_files = true; @@ -50,7 +51,7 @@ message CreateTaskAssignmentResponse { // When client (device) is accepted for the current task, this data structure // carries information necessary to begin task execution. -// Next Id: 9 +// Next Id: 11 message TaskAssignment { // population name. string population_name = 1; @@ -66,7 +67,8 @@ message TaskAssignment { // assignment id string assignment_id = 4; - // The name identifying the task that was assigned. + // The identifier of the task that was assigned. It's better to use the combination of + // `population_name` and `task_id` to identify a task. string task_name = 5; // The checkpoint from which to start execution. @@ -77,8 +79,19 @@ message TaskAssignment { // self uri string self_uri = 8; + + // The eligibility of the task. Device need to run this eligibility task + // to check if it's qualified to start the job before execute real task. + EligibilityTaskInfo eligibility_task_info = 9; + + // The signature of eligibility task. + Signature eligibility_task_info_signature = 10; } +// The signature information. Device can verify the signature to check integrity of +// the content. +message Signature {} + // Report result request. // The url to report result under v1 API is: // https://{host}/taskassignment/v1/population/{populationName}/task/{taskId}/aggregation/{aggregationId}/task-assignment/{assignmentId}:report-result @@ -93,6 +106,9 @@ message ReportResultRequest { // Failed. FAILED = 2; + + // Failed due to eligibility task is not qualified. + NOT_ELIGIBLE = 3; } Result result = 1; diff --git a/fcp/tensorflow/file_descriptor_filesystem.cc b/fcp/tensorflow/file_descriptor_filesystem.cc index 1659839..49747cf 100644 --- a/fcp/tensorflow/file_descriptor_filesystem.cc +++ b/fcp/tensorflow/file_descriptor_filesystem.cc @@ -1,3 +1,19 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include "fcp/tensorflow/file_descriptor_filesystem.h" #include <errno.h> diff --git a/fcp/tensorflow/file_descriptor_filesystem.h b/fcp/tensorflow/file_descriptor_filesystem.h index 86a4084..74b6652 100644 --- a/fcp/tensorflow/file_descriptor_filesystem.h +++ b/fcp/tensorflow/file_descriptor_filesystem.h @@ -1,3 +1,19 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #ifndef FCP_TENSORFLOW_FILE_DESCRIPTOR_FILESYSTEM_H_ #define FCP_TENSORFLOW_FILE_DESCRIPTOR_FILESYSTEM_H_ diff --git a/fcp/tensorflow/file_descriptor_filesystem_test.cc b/fcp/tensorflow/file_descriptor_filesystem_test.cc index 669f822..f21a2fa 100644 --- a/fcp/tensorflow/file_descriptor_filesystem_test.cc +++ b/fcp/tensorflow/file_descriptor_filesystem_test.cc @@ -1,4 +1,19 @@ - +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include "fcp/tensorflow/file_descriptor_filesystem.h" #include <fcntl.h> |