zone/OpenAssetTools
2
0
Fork 0
mirror of https://github.com/Laupetin/OpenAssetTools.git synced 2025-09-02 23:17:26 +00:00
Code Activity

Merge pull request #71 from diamante0018/main

Browse Source 
feat: add warning for dupe localize assets
This commit is contained in:
Jan
2023-12-31 12:38:56 +01:00
committed by GitHub
parent ef8442fa2b 196e2edd8a
commit 4eabf98712
28 changed files with 103 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
src/ObjLoading/Parsing/LocalizeFile/LocalizeFileParser.cpp
View File

@@ -1,38 +0,0 @@
#include "LocalizeFileParser.h"
#include "Sequence/SequenceLocalizeFileConfig.h"
#include "Sequence/SequenceLocalizeFileConsumeEmptyLines.h"
#include "Sequence/SequenceLocalizeFileEndMarker.h"
#include "Sequence/SequenceLocalizeFileLanguageValue.h"
#include "Sequence/SequenceLocalizeFileNotes.h"
#include "Sequence/SequenceLocalizeFileReference.h"
#include "Sequence/SequenceLocalizeFileVersion.h"
LocalizeFileParser::LocalizeFileParser(SimpleLexer* lexer, GameLanguage language)
: AbstractParser(lexer, std::make_unique<LocalizeFileParserState>(language))
{
}
const std::vector<AbstractParser<SimpleParserValue, LocalizeFileParserState>::sequence_t*>& LocalizeFileParser::GetTestsForState()
{
static std::vector<sequence_t*> tests({
new SequenceLocalizeFileReference(),
new SequenceLocalizeFileConfig(),
new SequenceLocalizeFileNotes(),
new SequenceLocalizeFileVersion(),
new SequenceLocalizeFileEndMarker(),
new SequenceLocalizeFileLanguageValue(),
new SequenceLocalizeFileConsumeEmptyLines(),
});
static std::vector<sequence_t*> noTests({
new SequenceLocalizeFileConsumeEmptyLines(),
});
return !m_state->m_end ? tests : noTests;
}
std::vector<LocalizeFileEntry> LocalizeFileParser::GetParsedValues()
{
return std::move(m_state->m_entries);
}

17
src/ObjLoading/Parsing/LocalizeFile/LocalizeFileParser.h
View File

@@ -1,17 +0,0 @@
#pragma once
#include "Game/GameLanguage.h"
#include "LocalizeFileParserState.h"
#include "Parsing/Impl/AbstractParser.h"
#include "Parsing/Simple/SimpleLexer.h"
#include "Parsing/Simple/SimpleParserValue.h"
class LocalizeFileParser final : public AbstractParser<SimpleParserValue, LocalizeFileParserState>
{
protected:
const std::vector<sequence_t*>& GetTestsForState() override;
public:
LocalizeFileParser(SimpleLexer* lexer, GameLanguage language);
std::vector<LocalizeFileEntry> GetParsedValues();
};

12
src/ObjLoading/Parsing/LocalizeFile/LocalizeFileParserState.cpp
View File

@@ -1,12 +0,0 @@
#include "LocalizeFileParserState.h"
#include "Localize/LocalizeCommon.h"
LocalizeFileParserState::LocalizeFileParserState(const GameLanguage language)
: m_end(false),
m_language(language)
{
m_language_name_caps = LocalizeCommon::GetNameOfLanguage(m_language);
for (auto& c : m_language_name_caps)
c = static_cast<char>(toupper(c));
}

22
src/ObjLoading/Parsing/LocalizeFile/LocalizeFileParserState.h
View File

@@ -1,22 +0,0 @@
#pragma once
#include "Game/GameLanguage.h"
#include "Localize/LocalizeFile.h"
#include <unordered_set>
#include <vector>
class LocalizeFileParserState
{
public:
bool m_end;
std::vector<LocalizeFileEntry> m_entries;
GameLanguage m_language;
std::string m_language_name_caps;
std::string m_current_reference;
std::unordered_set<std::string> m_current_reference_languages;
explicit LocalizeFileParserState(GameLanguage language);
};

47
src/ObjLoading/Parsing/LocalizeFile/LocalizeFileReader.cpp
View File

@@ -1,47 +0,0 @@
#include "LocalizeFileReader.h"
#include "LocalizeFileParser.h"
#include "Parsing/Impl/CommentRemovingStreamProxy.h"
#include "Parsing/Impl/ParserSingleInputStream.h"
LocalizeFileReader::LocalizeFileReader(std::istream& stream, std::string fileName, GameLanguage language)
: m_file_name(std::move(fileName)),
m_stream(nullptr),
m_language(language)
{
OpenBaseStream(stream);
SetupStreamProxies();
m_stream = m_open_streams.back().get();
}
bool LocalizeFileReader::OpenBaseStream(std::istream& stream)
{
m_open_streams.emplace_back(std::make_unique<ParserSingleInputStream>(stream, m_file_name));
return true;
}
void LocalizeFileReader::SetupStreamProxies()
{
m_open_streams.emplace_back(std::make_unique<CommentRemovingStreamProxy>(m_open_streams.back().get()));
m_stream = m_open_streams.back().get();
}
std::vector<LocalizeFileEntry> LocalizeFileReader::ReadLocalizeFile()
{
SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = true;
lexerConfig.m_read_strings = true;
lexerConfig.m_string_escape_sequences = true;
lexerConfig.m_read_integer_numbers = false;
lexerConfig.m_read_floating_point_numbers = false;
const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig));
const auto parser = std::make_unique<LocalizeFileParser>(lexer.get(), m_language);
if (parser->Parse())
return parser->GetParsedValues();
std::cout << "Parsing localization file failed!" << std::endl;
return std::vector<LocalizeFileEntry>();
}

25
src/ObjLoading/Parsing/LocalizeFile/LocalizeFileReader.h
View File

@@ -1,25 +0,0 @@
#pragma once
#include "Game/GameLanguage.h"
#include "Localize/LocalizeFile.h"
#include "Parsing/IParserLineStream.h"
#include <memory>
#include <string>
#include <vector>
class LocalizeFileReader
{
std::string m_file_name;
IParserLineStream* m_stream;
std::vector<std::unique_ptr<IParserLineStream>> m_open_streams;
GameLanguage m_language;
bool OpenBaseStream(std::istream& stream);
void SetupStreamProxies();
public:
LocalizeFileReader(std::istream& stream, std::string fileName, GameLanguage language);
std::vector<LocalizeFileEntry> ReadLocalizeFile();
};

16
src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileConfig.cpp
View File

@@ -1,16 +0,0 @@
#include "SequenceLocalizeFileConfig.h"
#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
SequenceLocalizeFileConfig::SequenceLocalizeFileConfig()
{
const SimpleMatcherFactory create(this);
AddMatchers({
create.Keyword("CONFIG"),
create.String(),
create.Type(SimpleParserValueType::NEW_LINE),
});
}
void SequenceLocalizeFileConfig::ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const {}

12
src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileConfig.h
View File

@@ -1,12 +0,0 @@
#pragma once
#include "Parsing/LocalizeFile/LocalizeFileParser.h"
class SequenceLocalizeFileConfig final : public LocalizeFileParser::sequence_t
{
protected:
void ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const override;
public:
SequenceLocalizeFileConfig();
};

14
src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileConsumeEmptyLines.cpp
View File

@@ -1,14 +0,0 @@
#include "SequenceLocalizeFileConsumeEmptyLines.h"
#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
SequenceLocalizeFileConsumeEmptyLines::SequenceLocalizeFileConsumeEmptyLines()
{
const SimpleMatcherFactory create(this);
AddMatchers({
create.Type(SimpleParserValueType::NEW_LINE),
});
}
void SequenceLocalizeFileConsumeEmptyLines::ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const {}

12
src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileConsumeEmptyLines.h
View File

@@ -1,12 +0,0 @@
#pragma once
#include "Parsing/LocalizeFile/LocalizeFileParser.h"
class SequenceLocalizeFileConsumeEmptyLines final : public LocalizeFileParser::sequence_t
{
protected:
void ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const override;
public:
SequenceLocalizeFileConsumeEmptyLines();
};

17
src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileEndMarker.cpp
View File

@@ -1,17 +0,0 @@
#include "SequenceLocalizeFileEndMarker.h"
#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
SequenceLocalizeFileEndMarker::SequenceLocalizeFileEndMarker()
{
const SimpleMatcherFactory create(this);
AddMatchers({
create.Keyword("ENDMARKER"),
});
}
void SequenceLocalizeFileEndMarker::ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const
{
state->m_end = true;
}

12
src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileEndMarker.h
View File

@@ -1,12 +0,0 @@
#pragma once
#include "Parsing/LocalizeFile/LocalizeFileParser.h"
class SequenceLocalizeFileEndMarker final : public LocalizeFileParser::sequence_t
{
protected:
void ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const override;
public:
SequenceLocalizeFileEndMarker();
};

35
src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileLanguageValue.cpp
View File

@@ -1,35 +0,0 @@
#include "SequenceLocalizeFileLanguageValue.h"
#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
#include <sstream>
SequenceLocalizeFileLanguageValue::SequenceLocalizeFileLanguageValue()
{
const SimpleMatcherFactory create(this);
AddMatchers({
create.KeywordPrefix("LANG_").Capture(CAPTURE_LANGUAGE_NAME),
create.String().Capture(CAPTURE_ENTRY_VALUE),
create.Type(SimpleParserValueType::NEW_LINE),
});
}
void SequenceLocalizeFileLanguageValue::ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const
{
const auto& langToken = result.NextCapture(CAPTURE_LANGUAGE_NAME);
const auto& valueToken = result.NextCapture(CAPTURE_ENTRY_VALUE);
const auto langName = langToken.IdentifierValue().substr(std::char_traits<char>::length("LANG_"));
const auto alreadyDefinedLanguage = state->m_current_reference_languages.find(langName);
if (alreadyDefinedLanguage != state->m_current_reference_languages.end())
{
std::ostringstream str;
str << "Value for reference \"" << state->m_current_reference << "\" already defined for language \"" << langToken.IdentifierValue() << "\"";
throw ParsingException(langToken.GetPos(), str.str());
}
state->m_current_reference_languages.emplace(langName);
if (langName == state->m_language_name_caps)
state->m_entries.emplace_back(state->m_current_reference, valueToken.StringValue());
}

15
src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileLanguageValue.h
View File

@@ -1,15 +0,0 @@
#pragma once
#include "Parsing/LocalizeFile/LocalizeFileParser.h"
class SequenceLocalizeFileLanguageValue final : public LocalizeFileParser::sequence_t
{
static constexpr auto CAPTURE_LANGUAGE_NAME = 1;
static constexpr auto CAPTURE_ENTRY_VALUE = 2;
protected:
void ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const override;
public:
SequenceLocalizeFileLanguageValue();
};

16
src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileNotes.cpp
View File

@@ -1,16 +0,0 @@
#include "SequenceLocalizeFileNotes.h"
#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
SequenceLocalizeFileNotes::SequenceLocalizeFileNotes()
{
const SimpleMatcherFactory create(this);
AddMatchers({
create.Keyword("FILENOTES"),
create.String(),
create.Type(SimpleParserValueType::NEW_LINE),
});
}
void SequenceLocalizeFileNotes::ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const {}

12
src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileNotes.h
View File

@@ -1,12 +0,0 @@
#pragma once
#include "Parsing/LocalizeFile/LocalizeFileParser.h"
class SequenceLocalizeFileNotes final : public LocalizeFileParser::sequence_t
{
protected:
void ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const override;
public:
SequenceLocalizeFileNotes();
};

26
src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileReference.cpp
View File

@@ -1,26 +0,0 @@
#include "SequenceLocalizeFileReference.h"
#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
SequenceLocalizeFileReference::SequenceLocalizeFileReference()
{
const SimpleMatcherFactory create(this);
AddMatchers({
create.Keyword("REFERENCE"),
create
.Or({
create.Identifier(),
create.String(),
})
.Capture(CAPTURE_REFERENCE_NAME),
create.Type(SimpleParserValueType::NEW_LINE),
});
}
void SequenceLocalizeFileReference::ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const
{
const auto& nameToken = result.NextCapture(CAPTURE_REFERENCE_NAME);
state->m_current_reference = nameToken.m_type == SimpleParserValueType::IDENTIFIER ? nameToken.IdentifierValue() : nameToken.StringValue();
state->m_current_reference_languages.clear();
}

14
src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileReference.h
View File

@@ -1,14 +0,0 @@
#pragma once
#include "Parsing/LocalizeFile/LocalizeFileParser.h"
class SequenceLocalizeFileReference final : public LocalizeFileParser::sequence_t
{
static constexpr auto CAPTURE_REFERENCE_NAME = 1;
protected:
void ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const override;
public:
SequenceLocalizeFileReference();
};

23
src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileVersion.cpp
View File

@@ -1,23 +0,0 @@
#include "SequenceLocalizeFileVersion.h"
#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
SequenceLocalizeFileVersion::SequenceLocalizeFileVersion()
{
const SimpleMatcherFactory create(this);
AddMatchers({
create.Keyword("VERSION"),
create.String().Capture(CAPTURE_VERSION),
create.Type(SimpleParserValueType::NEW_LINE),
});
}
void SequenceLocalizeFileVersion::ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const
{
const auto& versionCapture = result.NextCapture(CAPTURE_VERSION);
if (versionCapture.StringValue() != "1")
{
throw ParsingException(versionCapture.GetPos(), "Localize file needs to be version 1");
}
}

14
src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileVersion.h
View File

@@ -1,14 +0,0 @@
#pragma once
#include "Parsing/LocalizeFile/LocalizeFileParser.h"
class SequenceLocalizeFileVersion final : public LocalizeFileParser::sequence_t
{
static constexpr auto CAPTURE_VERSION = 1;
protected:
void ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const override;
public:
SequenceLocalizeFileVersion();
};