summaryrefslogtreecommitdiff
path: root/native/lang_id
diff options
context:
space:
mode:
Diffstat (limited to 'native/lang_id')
-rw-r--r--native/lang_id/common/embedding-feature-extractor.h17
-rw-r--r--native/lang_id/common/embedding-feature-interface.h3
-rw-r--r--native/lang_id/common/embedding-network.cc11
-rw-r--r--native/lang_id/common/fel/feature-descriptors.h9
-rw-r--r--native/lang_id/common/fel/feature-extractor.h11
-rw-r--r--native/lang_id/common/fel/feature-types.h16
-rw-r--r--native/lang_id/common/fel/fel-parser.cc5
-rw-r--r--native/lang_id/common/fel/fel-parser.h2
-rw-r--r--native/lang_id/common/fel/workspace.h5
-rw-r--r--native/lang_id/common/flatbuffers/model-utils.cc3
-rw-r--r--native/lang_id/common/flatbuffers/model-utils.h3
-rw-r--r--native/lang_id/common/math/algorithm.h4
-rw-r--r--native/lang_id/common/math/softmax.cc8
-rw-r--r--native/lang_id/features/char-ngram-feature.cc5
-rw-r--r--native/lang_id/features/char-ngram-feature.h1
-rw-r--r--native/lang_id/features/relevant-script-feature.cc1
-rw-r--r--native/lang_id/lang-id.cc2
17 files changed, 59 insertions, 47 deletions
diff --git a/native/lang_id/common/embedding-feature-extractor.h b/native/lang_id/common/embedding-feature-extractor.h
index ba4f858..8363321 100644
--- a/native/lang_id/common/embedding-feature-extractor.h
+++ b/native/lang_id/common/embedding-feature-extractor.h
@@ -25,6 +25,8 @@
#include "lang_id/common/fel/task-context.h"
#include "lang_id/common/fel/workspace.h"
#include "lang_id/common/lite_base/attributes.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/string_view.h"
namespace libtextclassifier3 {
namespace mobile {
@@ -46,7 +48,7 @@ class GenericEmbeddingFeatureExtractor {
//
// |arg_prefix| is a string prefix for the relevant TaskContext parameters, to
// avoid name clashes. See GetParamName().
- explicit GenericEmbeddingFeatureExtractor(const std::string &arg_prefix)
+ explicit GenericEmbeddingFeatureExtractor(absl::string_view arg_prefix)
: arg_prefix_(arg_prefix) {}
virtual ~GenericEmbeddingFeatureExtractor() {}
@@ -70,11 +72,8 @@ class GenericEmbeddingFeatureExtractor {
}
// Get parameter name by concatenating the prefix and the original name.
- std::string GetParamName(const std::string &param_name) const {
- std::string full_name = arg_prefix_;
- full_name.push_back('_');
- full_name.append(param_name);
- return full_name;
+ std::string GetParamName(absl::string_view param_name) const {
+ return absl::StrCat(arg_prefix_, "_", param_name);
}
private:
@@ -108,7 +107,7 @@ class EmbeddingFeatureExtractor : public GenericEmbeddingFeatureExtractor {
//
// |arg_prefix| is a string prefix for the relevant TaskContext parameters, to
// avoid name clashes. See GetParamName().
- explicit EmbeddingFeatureExtractor(const std::string &arg_prefix)
+ explicit EmbeddingFeatureExtractor(absl::string_view arg_prefix)
: GenericEmbeddingFeatureExtractor(arg_prefix) {}
// Sets up all predicate maps, feature extractors, and flags.
@@ -117,7 +116,7 @@ class EmbeddingFeatureExtractor : public GenericEmbeddingFeatureExtractor {
return false;
}
feature_extractors_.resize(embedding_fml().size());
- for (int i = 0; i < embedding_fml().size(); ++i) {
+ for (size_t i = 0; i < embedding_fml().size(); ++i) {
feature_extractors_[i].reset(new EXTRACTOR());
if (!feature_extractors_[i]->Parse(embedding_fml()[i])) return false;
if (!feature_extractors_[i]->Setup(context)) return false;
@@ -158,7 +157,7 @@ class EmbeddingFeatureExtractor : public GenericEmbeddingFeatureExtractor {
std::vector<FeatureVector> *features) const {
// DCHECK(features != nullptr);
// DCHECK_EQ(features->size(), feature_extractors_.size());
- for (int i = 0; i < feature_extractors_.size(); ++i) {
+ for (size_t i = 0; i < feature_extractors_.size(); ++i) {
(*features)[i].clear();
feature_extractors_[i]->ExtractFeatures(workspaces, obj, args...,
&(*features)[i]);
diff --git a/native/lang_id/common/embedding-feature-interface.h b/native/lang_id/common/embedding-feature-interface.h
index 75d0c98..26d574b 100644
--- a/native/lang_id/common/embedding-feature-interface.h
+++ b/native/lang_id/common/embedding-feature-interface.h
@@ -25,6 +25,7 @@
#include "lang_id/common/fel/task-context.h"
#include "lang_id/common/fel/workspace.h"
#include "lang_id/common/lite_base/attributes.h"
+#include "absl/strings/string_view.h"
namespace libtextclassifier3 {
namespace mobile {
@@ -36,7 +37,7 @@ class EmbeddingFeatureInterface {
//
// |arg_prefix| is a string prefix for the TaskContext parameters, passed to
// |the underlying EmbeddingFeatureExtractor.
- explicit EmbeddingFeatureInterface(const std::string &arg_prefix)
+ explicit EmbeddingFeatureInterface(absl::string_view arg_prefix)
: feature_extractor_(arg_prefix) {}
// Sets up feature extractors and flags for processing (inference).
diff --git a/native/lang_id/common/embedding-network.cc b/native/lang_id/common/embedding-network.cc
index 49c9ca0..0fe35d4 100644
--- a/native/lang_id/common/embedding-network.cc
+++ b/native/lang_id/common/embedding-network.cc
@@ -153,7 +153,7 @@ void EmbeddingNetwork::ConcatEmbeddings(
concat->resize(concat_layer_size_);
// "es_index" stands for "embedding space index".
- for (int es_index = 0; es_index < feature_vectors.size(); ++es_index) {
+ for (size_t es_index = 0; es_index < feature_vectors.size(); ++es_index) {
const int concat_offset = concat_offset_[es_index];
const EmbeddingNetworkParams::Matrix &embedding_matrix =
@@ -167,7 +167,8 @@ void EmbeddingNetwork::ConcatEmbeddings(
for (int fi = 0; fi < num_features; ++fi) {
const FeatureType *feature_type = feature_vector.type(fi);
int feature_offset = concat_offset + feature_type->base() * embedding_dim;
- SAFTM_CHECK_LE(feature_offset + embedding_dim, concat->size());
+ SAFTM_CHECK_LE(feature_offset + embedding_dim,
+ static_cast<int>(concat->size()));
// Weighted embeddings will be added starting from this address.
float *concat_ptr = concat->data() + feature_offset;
@@ -257,7 +258,7 @@ void EmbeddingNetwork::ComputeFinalScores(
ConcatEmbeddings(features, &input);
if (!extra_inputs.empty()) {
input.reserve(input.size() + extra_inputs.size());
- for (int i = 0; i < extra_inputs.size(); i++) {
+ for (size_t i = 0; i < extra_inputs.size(); i++) {
input.push_back(extra_inputs[i]);
}
}
@@ -281,8 +282,8 @@ void EmbeddingNetwork::ComputeFinalScores(
v_out = &(storage[i % 2]);
}
const bool apply_relu = i > 0;
- SparseReluProductPlusBias(
- apply_relu, layer_weights_[i], layer_bias_[i], *v_in, v_out);
+ SparseReluProductPlusBias(apply_relu, layer_weights_[i], layer_bias_[i],
+ *v_in, v_out);
v_in = v_out;
}
}
diff --git a/native/lang_id/common/fel/feature-descriptors.h b/native/lang_id/common/fel/feature-descriptors.h
index 3bdc2fa..f9536d9 100644
--- a/native/lang_id/common/fel/feature-descriptors.h
+++ b/native/lang_id/common/fel/feature-descriptors.h
@@ -24,6 +24,7 @@
#include "lang_id/common/lite_base/integral-types.h"
#include "lang_id/common/lite_base/logging.h"
#include "lang_id/common/lite_base/macros.h"
+#include "absl/strings/string_view.h"
namespace libtextclassifier3 {
namespace mobile {
@@ -33,10 +34,10 @@ class Parameter {
public:
Parameter() {}
- void set_name(const std::string &name) { name_ = name; }
+ void set_name(absl::string_view name) { name_ = std::string(name); }
const std::string &name() const { return name_; }
- void set_value(const std::string &value) { value_ = value; }
+ void set_value(absl::string_view value) { value_ = std::string(value); }
const std::string &value() const { return value_; }
private:
@@ -52,13 +53,13 @@ class FeatureFunctionDescriptor {
// Accessors for the feature function type. The function type is the string
// that the feature extractor code is registered under.
- void set_type(const std::string &type) { type_ = type; }
+ void set_type(absl::string_view type) { type_ = std::string(type); }
const std::string &type() const { return type_; }
// Accessors for the feature function name. The function name (if available)
// is used for some log messages. Otherwise, a more precise, but also more
// verbose name based on the feature specification is used.
- void set_name(const std::string &name) { name_ = name; }
+ void set_name(absl::string_view name) { name_ = std::string(name); }
const std::string &name() const { return name_; }
// Accessors for the default (name-less) parameter.
diff --git a/native/lang_id/common/fel/feature-extractor.h b/native/lang_id/common/fel/feature-extractor.h
index c09e1eb..805272b 100644
--- a/native/lang_id/common/fel/feature-extractor.h
+++ b/native/lang_id/common/fel/feature-extractor.h
@@ -52,6 +52,7 @@
#include "lang_id/common/lite_base/macros.h"
#include "lang_id/common/registry.h"
#include "lang_id/common/stl-util.h"
+#include "absl/strings/string_view.h"
namespace libtextclassifier3 {
namespace mobile {
@@ -261,7 +262,7 @@ class GenericFeatureFunction {
// Returns/sets/clears function name prefix.
const std::string &prefix() const { return prefix_; }
- void set_prefix(const std::string &prefix) { prefix_ = prefix; }
+ void set_prefix(absl::string_view prefix) { prefix_ = std::string(prefix); }
protected:
// Returns the feature type for single-type feature functions.
@@ -341,7 +342,7 @@ class FeatureFunction
// the relevant cc_library was not linked-in).
static Self *Instantiate(const GenericFeatureExtractor *extractor,
const FeatureFunctionDescriptor *fd,
- const std::string &prefix) {
+ absl::string_view prefix) {
Self *f = Self::Create(fd->type());
if (f != nullptr) {
f->set_extractor(extractor);
@@ -440,7 +441,7 @@ class NestedFeatureFunction : public FeatureFunction<OBJ, ARGS...> {
SAFTM_MUST_USE_RESULT static bool CreateNested(
const GenericFeatureExtractor *extractor,
const FeatureFunctionDescriptor *fd, std::vector<NES *> *functions,
- const std::string &prefix) {
+ absl::string_view prefix) {
for (int i = 0; i < fd->feature_size(); ++i) {
const FeatureFunctionDescriptor &sub = fd->feature(i);
NES *f = NES::Instantiate(extractor, &sub, prefix);
@@ -614,7 +615,7 @@ class FeatureExtractor : public GenericFeatureExtractor {
result->reserve(this->feature_types());
// Extract features.
- for (int i = 0; i < functions_.size(); ++i) {
+ for (size_t i = 0; i < functions_.size(); ++i) {
functions_[i]->Evaluate(workspaces, object, args..., result);
}
}
@@ -636,7 +637,7 @@ class FeatureExtractor : public GenericFeatureExtractor {
// Collect all feature types used in the feature extractor.
void GetFeatureTypes(std::vector<FeatureType *> *types) const override {
- for (int i = 0; i < functions_.size(); ++i) {
+ for (size_t i = 0; i < functions_.size(); ++i) {
functions_[i]->GetFeatureTypes(types);
}
}
diff --git a/native/lang_id/common/fel/feature-types.h b/native/lang_id/common/fel/feature-types.h
index ae422af..308ee90 100644
--- a/native/lang_id/common/fel/feature-types.h
+++ b/native/lang_id/common/fel/feature-types.h
@@ -27,6 +27,8 @@
#include "lang_id/common/lite_base/integral-types.h"
#include "lang_id/common/lite_base/logging.h"
#include "lang_id/common/lite_strings/str-cat.h"
+#include "absl/strings/match.h"
+#include "absl/strings/string_view.h"
namespace libtextclassifier3 {
namespace mobile {
@@ -44,10 +46,10 @@ typedef Predicate FeatureValue;
class FeatureType {
public:
// Initializes a feature type.
- explicit FeatureType(const std::string &name)
+ explicit FeatureType(absl::string_view name)
: name_(name),
base_(0),
- is_continuous_(name.find("continuous") != std::string::npos) {}
+ is_continuous_(absl::StrContains(name, "continuous")) {}
virtual ~FeatureType() {}
@@ -91,7 +93,7 @@ class FeatureType {
// };
class EnumFeatureType : public FeatureType {
public:
- EnumFeatureType(const std::string &name,
+ EnumFeatureType(absl::string_view name,
const std::map<FeatureValue, std::string> &value_names)
: FeatureType(name), value_names_(value_names) {
for (const auto &pair : value_names) {
@@ -127,8 +129,8 @@ class EnumFeatureType : public FeatureType {
// Feature type for binary features.
class BinaryFeatureType : public FeatureType {
public:
- BinaryFeatureType(const std::string &name, const std::string &off,
- const std::string &on)
+ BinaryFeatureType(absl::string_view name, absl::string_view off,
+ absl::string_view on)
: FeatureType(name), off_(off), on_(on) {}
// Returns the feature name for a given feature value.
@@ -151,7 +153,7 @@ class BinaryFeatureType : public FeatureType {
class NumericFeatureType : public FeatureType {
public:
// Initializes numeric feature.
- NumericFeatureType(const std::string &name, FeatureValue size)
+ NumericFeatureType(absl::string_view name, FeatureValue size)
: FeatureType(name), size_(size) {}
// Returns numeric feature value.
@@ -171,7 +173,7 @@ class NumericFeatureType : public FeatureType {
// Feature type for byte features, including an "outside" value.
class ByteFeatureType : public NumericFeatureType {
public:
- explicit ByteFeatureType(const std::string &name)
+ explicit ByteFeatureType(absl::string_view name)
: NumericFeatureType(name, 257) {}
std::string GetFeatureValueName(FeatureValue value) const override {
diff --git a/native/lang_id/common/fel/fel-parser.cc b/native/lang_id/common/fel/fel-parser.cc
index 2682941..c393ae8 100644
--- a/native/lang_id/common/fel/fel-parser.cc
+++ b/native/lang_id/common/fel/fel-parser.cc
@@ -22,6 +22,7 @@
#include "lang_id/common/lite_base/logging.h"
#include "lang_id/common/lite_strings/numbers.h"
+#include "absl/strings/string_view.h"
namespace libtextclassifier3 {
namespace mobile {
@@ -47,9 +48,9 @@ inline bool IsValidCharInsideNumber(char c) {
}
} // namespace
-bool FELParser::Initialize(const std::string &source) {
+bool FELParser::Initialize(absl::string_view source) {
// Initialize parser state.
- source_ = source;
+ source_ = std::string(source);
current_ = source_.begin();
item_start_ = line_start_ = current_;
line_number_ = item_line_number_ = 1;
diff --git a/native/lang_id/common/fel/fel-parser.h b/native/lang_id/common/fel/fel-parser.h
index d2c454c..bb33eaf 100644
--- a/native/lang_id/common/fel/fel-parser.h
+++ b/native/lang_id/common/fel/fel-parser.h
@@ -58,7 +58,7 @@ class FELParser {
private:
// Initializes the parser with the source text.
// Returns true on success, false on syntax error.
- bool Initialize(const std::string &source);
+ bool Initialize(absl::string_view source);
// Outputs an error message, with context info.
void ReportError(const std::string &error_message);
diff --git a/native/lang_id/common/fel/workspace.h b/native/lang_id/common/fel/workspace.h
index 2ac5b26..71d1550 100644
--- a/native/lang_id/common/fel/workspace.h
+++ b/native/lang_id/common/fel/workspace.h
@@ -31,6 +31,7 @@
#include "lang_id/common/lite_base/logging.h"
#include "lang_id/common/lite_base/macros.h"
+#include "absl/strings/string_view.h"
namespace libtextclassifier3 {
namespace mobile {
@@ -71,7 +72,7 @@ class WorkspaceRegistry {
// Returns the index of a named workspace, adding it to the registry first
// if necessary.
template <class W>
- int Request(const std::string &name) {
+ int Request(absl::string_view name) {
const int id = TypeId<W>::type_id;
max_workspace_id_ = std::max(id, max_workspace_id_);
workspace_types_[id] = W::TypeName();
@@ -79,7 +80,7 @@ class WorkspaceRegistry {
for (int i = 0; i < names.size(); ++i) {
if (names[i] == name) return i;
}
- names.push_back(name);
+ names.push_back(std::string(name));
return names.size() - 1;
}
diff --git a/native/lang_id/common/flatbuffers/model-utils.cc b/native/lang_id/common/flatbuffers/model-utils.cc
index 8efa386..592f616 100644
--- a/native/lang_id/common/flatbuffers/model-utils.cc
+++ b/native/lang_id/common/flatbuffers/model-utils.cc
@@ -22,6 +22,7 @@
#include "lang_id/common/lite_base/logging.h"
#include "lang_id/common/math/checksum.h"
+#include "absl/strings/string_view.h"
namespace libtextclassifier3 {
namespace saft_fbs {
@@ -64,7 +65,7 @@ const Model *GetVerifiedModelFromBytes(const char *data, size_t num_bytes) {
return model;
}
-const ModelInput *GetInputByName(const Model *model, const std::string &name) {
+const ModelInput *GetInputByName(const Model *model, absl::string_view name) {
if (model == nullptr) {
SAFTM_LOG(ERROR) << "GetInputByName called with model == nullptr";
return nullptr;
diff --git a/native/lang_id/common/flatbuffers/model-utils.h b/native/lang_id/common/flatbuffers/model-utils.h
index cf33dd5..16494b1 100644
--- a/native/lang_id/common/flatbuffers/model-utils.h
+++ b/native/lang_id/common/flatbuffers/model-utils.h
@@ -25,6 +25,7 @@
#include "lang_id/common/flatbuffers/model_generated.h"
#include "lang_id/common/lite_base/integral-types.h"
#include "lang_id/common/lite_strings/stringpiece.h"
+#include "absl/strings/string_view.h"
namespace libtextclassifier3 {
namespace saft_fbs {
@@ -46,7 +47,7 @@ inline const Model *GetVerifiedModelFromBytes(mobile::StringPiece bytes) {
// Returns the |model| input with specified |name|. Returns nullptr if no such
// input exists. If |model| contains multiple inputs with that |name|, returns
// the first one (model builders should avoid building such models).
-const ModelInput *GetInputByName(const Model *model, const std::string &name);
+const ModelInput *GetInputByName(const Model *model, absl::string_view name);
// Returns a StringPiece pointing to the bytes for the content of |input|. In
// case of errors, returns StringPiece(nullptr, 0).
diff --git a/native/lang_id/common/math/algorithm.h b/native/lang_id/common/math/algorithm.h
index 5c8596b..e2f7179 100644
--- a/native/lang_id/common/math/algorithm.h
+++ b/native/lang_id/common/math/algorithm.h
@@ -81,7 +81,7 @@ std::vector<int> GetTopKIndices(int k, const std::vector<T> &v,
return std::vector<int>();
}
- if (k > v.size()) {
+ if (static_cast<size_t>(k) > v.size()) {
k = v.size();
}
@@ -114,7 +114,7 @@ std::vector<int> GetTopKIndices(int k, const std::vector<T> &v,
// indicated by the indices from |heap|.
//
// Invariant C: |heap| is a max heap, according to order rev_vcomp.
- for (int i = k; i < v.size(); ++i) {
+ for (size_t i = k; i < v.size(); ++i) {
// We have to update |heap| iff v[i] is larger than the smallest of the
// top-k seen so far. This test is easy to do, due to Invariant B above.
if (smaller(v[heap[0]], v[i])) {
diff --git a/native/lang_id/common/math/softmax.cc b/native/lang_id/common/math/softmax.cc
index 249ed57..c2f7e89 100644
--- a/native/lang_id/common/math/softmax.cc
+++ b/native/lang_id/common/math/softmax.cc
@@ -26,7 +26,7 @@ namespace libtextclassifier3 {
namespace mobile {
float ComputeSoftmaxProbability(const std::vector<float> &scores, int label) {
- if ((label < 0) || (label >= scores.size())) {
+ if ((label < 0) || (static_cast<size_t>(label) >= scores.size())) {
SAFTM_LOG(ERROR) << "label " << label << " outside range "
<< "[0, " << scores.size() << ")";
return 0.0f;
@@ -43,8 +43,8 @@ float ComputeSoftmaxProbability(const std::vector<float> &scores, int label) {
// which saves two calls to exp().
const float label_score = scores[label];
float denominator = 1.0f; // Contribution of i == label.
- for (int i = 0; i < scores.size(); ++i) {
- if (i == label) continue;
+ for (size_t i = 0; i < scores.size(); ++i) {
+ if (static_cast<int>(i) == label) continue;
const float delta_score = scores[i] - label_score;
// TODO(salcianu): one can optimize the test below, to avoid any float
@@ -94,7 +94,7 @@ std::vector<float> ComputeSoftmax(const std::vector<float> &scores,
denominator += exp_score;
}
- for (int i = 0; i < scores.size(); ++i) {
+ for (size_t i = 0; i < scores.size(); ++i) {
softmax.push_back(exp_scores[i] / denominator);
}
return softmax;
diff --git a/native/lang_id/features/char-ngram-feature.cc b/native/lang_id/features/char-ngram-feature.cc
index 31faf2f..c367f71 100644
--- a/native/lang_id/features/char-ngram-feature.cc
+++ b/native/lang_id/features/char-ngram-feature.cc
@@ -16,6 +16,7 @@
#include "lang_id/features/char-ngram-feature.h"
+#include <mutex>
#include <string>
#include <utility>
#include <vector>
@@ -64,8 +65,8 @@ bool ContinuousBagOfNgramsFunction::Init(TaskContext *context) {
int ContinuousBagOfNgramsFunction::ComputeNgramCounts(
const LightSentence &sentence) const {
- SAFTM_CHECK_EQ(counts_.size(), ngram_id_dimension_);
- SAFTM_CHECK_EQ(non_zero_count_indices_.size(), 0);
+ SAFTM_CHECK_EQ(static_cast<int>(counts_.size()), ngram_id_dimension_);
+ SAFTM_CHECK_EQ(non_zero_count_indices_.size(), 0u);
int total_count = 0;
diff --git a/native/lang_id/features/char-ngram-feature.h b/native/lang_id/features/char-ngram-feature.h
index db0f83e..5b16e30 100644
--- a/native/lang_id/features/char-ngram-feature.h
+++ b/native/lang_id/features/char-ngram-feature.h
@@ -19,6 +19,7 @@
#include <mutex> // NOLINT: see comments for state_mutex_
#include <string>
+#include <vector>
#include "lang_id/common/fel/feature-extractor.h"
#include "lang_id/common/fel/task-context.h"
diff --git a/native/lang_id/features/relevant-script-feature.cc b/native/lang_id/features/relevant-script-feature.cc
index e88b328..f24c23c 100644
--- a/native/lang_id/features/relevant-script-feature.cc
+++ b/native/lang_id/features/relevant-script-feature.cc
@@ -17,6 +17,7 @@
#include "lang_id/features/relevant-script-feature.h"
#include <string>
+#include <vector>
#include "lang_id/common/fel/feature-types.h"
#include "lang_id/common/fel/task-context.h"
diff --git a/native/lang_id/lang-id.cc b/native/lang_id/lang-id.cc
index f7c66f7..67de5fe 100644
--- a/native/lang_id/lang-id.cc
+++ b/native/lang_id/lang-id.cc
@@ -243,7 +243,7 @@ class LangIdImpl {
// Returns language code for a softmax label. See comments for languages_
// field. If label is out of range, returns LangId::kUnknownLanguageCode.
std::string GetLanguageForSoftmaxLabel(int label) const {
- if ((label >= 0) && (label < languages_.size())) {
+ if ((label >= 0) && (static_cast<size_t>(label) < languages_.size())) {
return languages_[label];
} else {
SAFTM_LOG(ERROR) << "Softmax label " << label << " outside range [0, "
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-20 15:31:29 +0000