Merge pull request #71 from diamante0018/main

feat: add warning for dupe localize assets

src/ObjCommon/Localize/LocalizeFile.cpp

@@ -1,9 +0,0 @@
-#include "LocalizeFile.h"
-
-LocalizeFileEntry::LocalizeFileEntry() = default;
-
-LocalizeFileEntry::LocalizeFileEntry(std::string key, std::string value)
-    : m_key(std::move(key)),
-      m_value(std::move(value))
-{
-}

src/ObjCommon/Localize/LocalizeFile.h

@@ -1,12 +0,0 @@
-#pragma once
-#include <string>
-
-class LocalizeFileEntry
-{
-public:
-    std::string m_key;
-    std::string m_value;
-
-    LocalizeFileEntry();
-    LocalizeFileEntry(std::string key, std::string value);
-};

src/ObjLoading/Game/IW4/AssetLoaders/AssetLoaderLocalizeEntry.cpp

@@ -1,7 +1,7 @@
 #include "AssetLoaderLocalizeEntry.h"
 
 #include "Localize/LocalizeCommon.h"
-#include "Parsing/LocalizeFile/LocalizeFileReader.h"
+#include "Localize/Parsing/LocalizeFileReader.h"
 
 #include <sstream>
 
@@ -36,16 +36,17 @@ bool AssetLoaderLocalizeEntry::LoadFromRaw(
     if (!file.IsOpen())
         return false;
 
-    LocalizeFileReader reader(*file.m_stream, assetName, zone->m_language);
+    auto* zoneState = manager->GetAssetLoadingContext()->GetZoneAssetLoaderState<LocalizeReadingZoneState>();
+    LocalizeFileReader reader(*file.m_stream, assetName, zone->m_language, zoneState);
     const auto localizeEntries = reader.ReadLocalizeFile();
 
-    for (const auto& entry : localizeEntries)
+    for (const auto& [key, value] : localizeEntries)
     {
         auto* localizeEntry = memory->Create<LocalizeEntry>();
-        localizeEntry->name = memory->Dup(entry.m_key.c_str());
-        localizeEntry->value = memory->Dup(entry.m_value.c_str());
+        localizeEntry->name = memory->Dup(key.c_str());
+        localizeEntry->value = memory->Dup(value.c_str());
 
-        manager->AddAsset(ASSET_TYPE_LOCALIZE_ENTRY, entry.m_key, localizeEntry);
+        manager->AddAsset(ASSET_TYPE_LOCALIZE_ENTRY, key, localizeEntry);
     }
 
     return true;

src/ObjLoading/Game/IW5/AssetLoaders/AssetLoaderLocalizeEntry.cpp

@@ -1,7 +1,7 @@
 #include "AssetLoaderLocalizeEntry.h"
 
 #include "Localize/LocalizeCommon.h"
-#include "Parsing/LocalizeFile/LocalizeFileReader.h"
+#include "Localize/Parsing/LocalizeFileReader.h"
 
 #include <sstream>
 
@@ -36,16 +36,17 @@ bool AssetLoaderLocalizeEntry::LoadFromRaw(
     if (!file.IsOpen())
         return false;
 
-    LocalizeFileReader reader(*file.m_stream, assetName, zone->m_language);
+    auto* zoneState = manager->GetAssetLoadingContext()->GetZoneAssetLoaderState<LocalizeReadingZoneState>();
+    LocalizeFileReader reader(*file.m_stream, assetName, zone->m_language, zoneState);
     const auto localizeEntries = reader.ReadLocalizeFile();
 
-    for (const auto& entry : localizeEntries)
+    for (const auto& [key, value] : localizeEntries)
     {
         auto* localizeEntry = memory->Create<LocalizeEntry>();
-        localizeEntry->name = memory->Dup(entry.m_key.c_str());
-        localizeEntry->value = memory->Dup(entry.m_value.c_str());
+        localizeEntry->name = memory->Dup(key.c_str());
+        localizeEntry->value = memory->Dup(value.c_str());
 
-        manager->AddAsset(ASSET_TYPE_LOCALIZE_ENTRY, entry.m_key, localizeEntry);
+        manager->AddAsset(ASSET_TYPE_LOCALIZE_ENTRY, key, localizeEntry);
     }
 
     return true;

src/ObjLoading/Game/T5/AssetLoaders/AssetLoaderLocalizeEntry.cpp

@@ -1,7 +1,7 @@
 #include "AssetLoaderLocalizeEntry.h"
 
 #include "Localize/LocalizeCommon.h"
-#include "Parsing/LocalizeFile/LocalizeFileReader.h"
+#include "Localize/Parsing/LocalizeFileReader.h"
 
 #include <sstream>
 
@@ -36,16 +36,17 @@ bool AssetLoaderLocalizeEntry::LoadFromRaw(
     if (!file.IsOpen())
         return false;
 
-    LocalizeFileReader reader(*file.m_stream, assetName, zone->m_language);
+    auto* zoneState = manager->GetAssetLoadingContext()->GetZoneAssetLoaderState<LocalizeReadingZoneState>();
+    LocalizeFileReader reader(*file.m_stream, assetName, zone->m_language, zoneState);
     const auto localizeEntries = reader.ReadLocalizeFile();
 
-    for (const auto& entry : localizeEntries)
+    for (const auto& [key, value] : localizeEntries)
     {
         auto* localizeEntry = memory->Create<LocalizeEntry>();
-        localizeEntry->name = memory->Dup(entry.m_key.c_str());
-        localizeEntry->value = memory->Dup(entry.m_value.c_str());
+        localizeEntry->name = memory->Dup(key.c_str());
+        localizeEntry->value = memory->Dup(value.c_str());
 
-        manager->AddAsset(ASSET_TYPE_LOCALIZE_ENTRY, entry.m_key, localizeEntry);
+        manager->AddAsset(ASSET_TYPE_LOCALIZE_ENTRY, key, localizeEntry);
     }
 
     return true;

src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderLocalizeEntry.cpp

@@ -1,7 +1,7 @@
 #include "AssetLoaderLocalizeEntry.h"
 
 #include "Localize/LocalizeCommon.h"
-#include "Parsing/LocalizeFile/LocalizeFileReader.h"
+#include "Localize/Parsing/LocalizeFileReader.h"
 
 #include <sstream>
 
@@ -36,16 +36,17 @@ bool AssetLoaderLocalizeEntry::LoadFromRaw(
     if (!file.IsOpen())
         return false;
 
-    LocalizeFileReader reader(*file.m_stream, assetName, zone->m_language);
+    auto* zoneState = manager->GetAssetLoadingContext()->GetZoneAssetLoaderState<LocalizeReadingZoneState>();
+    LocalizeFileReader reader(*file.m_stream, assetName, zone->m_language, zoneState);
     const auto localizeEntries = reader.ReadLocalizeFile();
 
-    for (const auto& entry : localizeEntries)
+    for (const auto& [key, value] : localizeEntries)
     {
         auto* localizeEntry = memory->Create<LocalizeEntry>();
-        localizeEntry->name = memory->Dup(entry.m_key.c_str());
-        localizeEntry->value = memory->Dup(entry.m_value.c_str());
+        localizeEntry->name = memory->Dup(key.c_str());
+        localizeEntry->value = memory->Dup(value.c_str());
 
-        manager->AddAsset(ASSET_TYPE_LOCALIZE_ENTRY, entry.m_key, localizeEntry);
+        manager->AddAsset(ASSET_TYPE_LOCALIZE_ENTRY, key, localizeEntry);
     }
 
     return true;

src/ObjLoading/Localize/LocalizeReadingZoneState.cpp

@@ -0,0 +1,11 @@
+#include "LocalizeReadingZoneState.h"
+
+bool LocalizeReadingZoneState::DoLocalizeEntryDuplicateCheck(const std::string& key)
+{
+    const auto existingEntry = m_keys.find(key);
+    if (existingEntry != m_keys.end())
+        return false;
+
+    m_keys.emplace(key);
+    return true;
+}

src/ObjLoading/Localize/LocalizeReadingZoneState.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include "AssetLoading/IZoneAssetLoaderState.h"
+
+#include <string>
+#include <unordered_set>
+
+class LocalizeReadingZoneState final : public IZoneAssetLoaderState
+{
+public:
+    /**
+     * Checks whether a localize key was already added.
+     * Inserts key if it was not added yet.
+     *
+     * \param key The key to check
+     * \returns \c true if key was not duplicated yet, \c false otherwise
+     */
+    bool DoLocalizeEntryDuplicateCheck(const std::string& key);
+
+private:
+    std::unordered_set<std::string> m_keys;
+};

src/ObjLoading/Localize/Parsing/LocalizeFileParser.cpp

@@ -8,8 +8,8 @@
 #include "Sequence/SequenceLocalizeFileReference.h"
 #include "Sequence/SequenceLocalizeFileVersion.h"
 
-LocalizeFileParser::LocalizeFileParser(SimpleLexer* lexer, GameLanguage language)
-    : AbstractParser(lexer, std::make_unique<LocalizeFileParserState>(language))
+LocalizeFileParser::LocalizeFileParser(SimpleLexer* lexer, GameLanguage language, LocalizeReadingZoneState* zoneState)
+    : AbstractParser(lexer, std::make_unique<LocalizeFileParserState>(language, zoneState))
 {
 }
 
@@ -32,7 +32,7 @@ const std::vector<AbstractParser<SimpleParserValue, LocalizeFileParserState>::se
     return !m_state->m_end ? tests : noTests;
 }
 
-std::vector<LocalizeFileEntry> LocalizeFileParser::GetParsedValues()
+std::map<std::string, std::string> LocalizeFileParser::GetParsedValues()
 {
     return std::move(m_state->m_entries);
 }

src/ObjLoading/Localize/Parsing/LocalizeFileParser.h

@@ -12,6 +12,6 @@ protected:
     const std::vector<sequence_t*>& GetTestsForState() override;
 
 public:
-    LocalizeFileParser(SimpleLexer* lexer, GameLanguage language);
-    std::vector<LocalizeFileEntry> GetParsedValues();
+    LocalizeFileParser(SimpleLexer* lexer, GameLanguage language, LocalizeReadingZoneState* zoneState);
+    std::map<std::string, std::string> GetParsedValues();
 };

src/ObjLoading/Localize/Parsing/LocalizeFileParserState.cpp

@@ -2,9 +2,10 @@
 
 #include "Localize/LocalizeCommon.h"
 
-LocalizeFileParserState::LocalizeFileParserState(const GameLanguage language)
+LocalizeFileParserState::LocalizeFileParserState(const GameLanguage language, LocalizeReadingZoneState* zoneState)
     : m_end(false),
-      m_language(language)
+      m_language(language),
+      m_zone_state(zoneState)
 {
     m_language_name_caps = LocalizeCommon::GetNameOfLanguage(m_language);
     for (auto& c : m_language_name_caps)

src/ObjLoading/Localize/Parsing/LocalizeFileParserState.h

@@ -1,22 +1,24 @@
 #pragma once
-#include "Game/GameLanguage.h"
-#include "Localize/LocalizeFile.h"
 
+#include "Game/GameLanguage.h"
+#include "Localize/LocalizeReadingZoneState.h"
+
+#include <map>
 #include <unordered_set>
-#include <vector>
 
 class LocalizeFileParserState
 {
 public:
     bool m_end;
 
-    std::vector<LocalizeFileEntry> m_entries;
+    std::map<std::string, std::string> m_entries;
 
     GameLanguage m_language;
+    LocalizeReadingZoneState* m_zone_state;
     std::string m_language_name_caps;
 
     std::string m_current_reference;
     std::unordered_set<std::string> m_current_reference_languages;
 
-    explicit LocalizeFileParserState(GameLanguage language);
+    LocalizeFileParserState(GameLanguage language, LocalizeReadingZoneState* zoneState);
 };

src/ObjLoading/Localize/Parsing/LocalizeFileReader.cpp

@@ -4,10 +4,11 @@
 #include "Parsing/Impl/CommentRemovingStreamProxy.h"
 #include "Parsing/Impl/ParserSingleInputStream.h"
 
-LocalizeFileReader::LocalizeFileReader(std::istream& stream, std::string fileName, GameLanguage language)
+LocalizeFileReader::LocalizeFileReader(std::istream& stream, std::string fileName, GameLanguage language, LocalizeReadingZoneState* zoneState)
     : m_file_name(std::move(fileName)),
       m_stream(nullptr),
-      m_language(language)
+      m_language(language),
+      m_zone_state(zoneState)
 {
     OpenBaseStream(stream);
     SetupStreamProxies();
@@ -27,7 +28,7 @@ void LocalizeFileReader::SetupStreamProxies()
     m_stream = m_open_streams.back().get();
 }
 
-std::vector<LocalizeFileEntry> LocalizeFileReader::ReadLocalizeFile()
+std::map<std::string, std::string> LocalizeFileReader::ReadLocalizeFile()
 {
     SimpleLexer::Config lexerConfig;
     lexerConfig.m_emit_new_line_tokens = true;
@@ -37,11 +38,11 @@ std::vector<LocalizeFileEntry> LocalizeFileReader::ReadLocalizeFile()
     lexerConfig.m_read_floating_point_numbers = false;
     const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig));
 
-    const auto parser = std::make_unique<LocalizeFileParser>(lexer.get(), m_language);
+    const auto parser = std::make_unique<LocalizeFileParser>(lexer.get(), m_language, m_zone_state);
 
     if (parser->Parse())
         return parser->GetParsedValues();
 
     std::cout << "Parsing localization file failed!" << std::endl;
-    return std::vector<LocalizeFileEntry>();
+    return std::map<std::string, std::string>();
 }

src/ObjLoading/Localize/Parsing/LocalizeFileReader.h

@@ -1,9 +1,10 @@
 #pragma once
 
 #include "Game/GameLanguage.h"
-#include "Localize/LocalizeFile.h"
+#include "Localize/LocalizeReadingZoneState.h"
 #include "Parsing/IParserLineStream.h"
 
+#include <map>
 #include <memory>
 #include <string>
 #include <vector>
@@ -14,12 +15,13 @@ class LocalizeFileReader
     IParserLineStream* m_stream;
     std::vector<std::unique_ptr<IParserLineStream>> m_open_streams;
     GameLanguage m_language;
+    LocalizeReadingZoneState* m_zone_state;
 
     bool OpenBaseStream(std::istream& stream);
     void SetupStreamProxies();
 
 public:
-    LocalizeFileReader(std::istream& stream, std::string fileName, GameLanguage language);
+    LocalizeFileReader(std::istream& stream, std::string fileName, GameLanguage language, LocalizeReadingZoneState* zoneState);
 
-    std::vector<LocalizeFileEntry> ReadLocalizeFile();
+    std::map<std::string, std::string> ReadLocalizeFile();
 };

src/ObjLoading/Localize/Parsing/Sequence/SequenceLocalizeFileConfig.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "Parsing/LocalizeFile/LocalizeFileParser.h"
+#include "Localize/Parsing/LocalizeFileParser.h"
 
 class SequenceLocalizeFileConfig final : public LocalizeFileParser::sequence_t
 {

src/ObjLoading/Localize/Parsing/Sequence/SequenceLocalizeFileConsumeEmptyLines.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "Parsing/LocalizeFile/LocalizeFileParser.h"
+#include "Localize/Parsing/LocalizeFileParser.h"
 
 class SequenceLocalizeFileConsumeEmptyLines final : public LocalizeFileParser::sequence_t
 {

src/ObjLoading/Localize/Parsing/Sequence/SequenceLocalizeFileEndMarker.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "Parsing/LocalizeFile/LocalizeFileParser.h"
+#include "Localize/Parsing/LocalizeFileParser.h"
 
 class SequenceLocalizeFileEndMarker final : public LocalizeFileParser::sequence_t
 {

src/ObjLoading/Localize/Parsing/Sequence/SequenceLocalizeFileLanguageValue.cpp

@@ -31,5 +31,13 @@ void SequenceLocalizeFileLanguageValue::ProcessMatch(LocalizeFileParserState* st
     state->m_current_reference_languages.emplace(langName);
 
     if (langName == state->m_language_name_caps)
-        state->m_entries.emplace_back(state->m_current_reference, valueToken.StringValue());
+    {
+        const auto& currentReference = state->m_current_reference;
+        if (!state->m_zone_state->DoLocalizeEntryDuplicateCheck(currentReference))
+        {
+            std::cout << "Localize: a value for reference \"" << currentReference << "\" was already defined\n";
+        }
+
+        state->m_entries[currentReference] = valueToken.StringValue();
+    }
 }

src/ObjLoading/Localize/Parsing/Sequence/SequenceLocalizeFileLanguageValue.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "Parsing/LocalizeFile/LocalizeFileParser.h"
+#include "Localize/Parsing/LocalizeFileParser.h"
 
 class SequenceLocalizeFileLanguageValue final : public LocalizeFileParser::sequence_t
 {

src/ObjLoading/Localize/Parsing/Sequence/SequenceLocalizeFileNotes.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "Parsing/LocalizeFile/LocalizeFileParser.h"
+#include "Localize/Parsing/LocalizeFileParser.h"
 
 class SequenceLocalizeFileNotes final : public LocalizeFileParser::sequence_t
 {

src/ObjLoading/Localize/Parsing/Sequence/SequenceLocalizeFileReference.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "Parsing/LocalizeFile/LocalizeFileParser.h"
+#include "Localize/Parsing/LocalizeFileParser.h"
 
 class SequenceLocalizeFileReference final : public LocalizeFileParser::sequence_t
 {

src/ObjLoading/Localize/Parsing/Sequence/SequenceLocalizeFileVersion.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "Parsing/LocalizeFile/LocalizeFileParser.h"
+#include "Localize/Parsing/LocalizeFileParser.h"
 
 class SequenceLocalizeFileVersion final : public LocalizeFileParser::sequence_t
 {