diff --git a/Modules.make b/Modules.make index 9f2a39dbd..b40c82a2c 100644 --- a/Modules.make +++ b/Modules.make @@ -65,7 +65,6 @@ MODULES += MODULE_MM_DT # ****** Neural Network ****** MODULES += MODULE_NN MODULES += MODULE_NN_SEQUENCE_TRAINING -MODULES += MODULE_THEANO_INTERFACE MODULES += MODULE_PYTHON # ****** OpenFst ****** diff --git a/apptainer/2022-10-21_tensorflow-1.15_arm_v1/makefiles/Modules.make b/apptainer/2022-10-21_tensorflow-1.15_arm_v1/makefiles/Modules.make index 765fd3849..7910cd240 100644 --- a/apptainer/2022-10-21_tensorflow-1.15_arm_v1/makefiles/Modules.make +++ b/apptainer/2022-10-21_tensorflow-1.15_arm_v1/makefiles/Modules.make @@ -65,7 +65,6 @@ MODULES += MODULE_MM_DT # ****** Neural Network ****** MODULES += MODULE_NN MODULES += MODULE_NN_SEQUENCE_TRAINING -MODULES += MODULE_THEANO_INTERFACE MODULES += MODULE_PYTHON # ****** OpenFst ****** diff --git a/apptainer/2022-10-21_tensorflow-1.15_v1/makefiles/Modules.make b/apptainer/2022-10-21_tensorflow-1.15_v1/makefiles/Modules.make index 765fd3849..7910cd240 100644 --- a/apptainer/2022-10-21_tensorflow-1.15_v1/makefiles/Modules.make +++ b/apptainer/2022-10-21_tensorflow-1.15_v1/makefiles/Modules.make @@ -65,7 +65,6 @@ MODULES += MODULE_MM_DT # ****** Neural Network ****** MODULES += MODULE_NN MODULES += MODULE_NN_SEQUENCE_TRAINING -MODULES += MODULE_THEANO_INTERFACE MODULES += MODULE_PYTHON # ****** OpenFst ****** diff --git a/apptainer/2023-05-08_tensorflow-2.8_v1/makefiles/Modules.make b/apptainer/2023-05-08_tensorflow-2.8_v1/makefiles/Modules.make index 7a5b1d1ee..2d9750913 100644 --- a/apptainer/2023-05-08_tensorflow-2.8_v1/makefiles/Modules.make +++ b/apptainer/2023-05-08_tensorflow-2.8_v1/makefiles/Modules.make @@ -65,7 +65,6 @@ MODULES += MODULE_MM_DT # ****** Neural Network ****** MODULES += MODULE_NN MODULES += MODULE_NN_SEQUENCE_TRAINING -MODULES += MODULE_THEANO_INTERFACE MODULES += MODULE_PYTHON # ****** OpenFst ****** diff --git a/apptainer/2023-08-09_tensorflow-2.8_onnx-1.15_v1/makefiles/Modules.make b/apptainer/2023-08-09_tensorflow-2.8_onnx-1.15_v1/makefiles/Modules.make index 56b50dbbe..827395ac2 100644 --- a/apptainer/2023-08-09_tensorflow-2.8_onnx-1.15_v1/makefiles/Modules.make +++ b/apptainer/2023-08-09_tensorflow-2.8_onnx-1.15_v1/makefiles/Modules.make @@ -66,7 +66,6 @@ MODULES += MODULE_MM_DT # ****** Neural Network ****** MODULES += MODULE_NN MODULES += MODULE_NN_SEQUENCE_TRAINING -MODULES += MODULE_THEANO_INTERFACE MODULES += MODULE_PYTHON # ****** OpenFst ****** diff --git a/apptainer/2023-11-08_tensorflow-2.14_v1/makefiles/Modules.make b/apptainer/2023-11-08_tensorflow-2.14_v1/makefiles/Modules.make index 7a45546db..b92813ac3 100644 --- a/apptainer/2023-11-08_tensorflow-2.14_v1/makefiles/Modules.make +++ b/apptainer/2023-11-08_tensorflow-2.14_v1/makefiles/Modules.make @@ -65,7 +65,6 @@ MODULES += MODULE_MM_DT # ****** Neural Network ****** MODULES += MODULE_NN MODULES += MODULE_NN_SEQUENCE_TRAINING -MODULES += MODULE_THEANO_INTERFACE MODULES += MODULE_PYTHON # ****** OpenFst ****** diff --git a/src/Bliss/CorpusDescription.cc b/src/Bliss/CorpusDescription.cc index 0e54b0aac..291a71039 100644 --- a/src/Bliss/CorpusDescription.cc +++ b/src/Bliss/CorpusDescription.cc @@ -26,9 +26,6 @@ #include #include -#ifdef MODULE_THEANO_INTERFACE -#include "TheanoSegmentOrderingVisitor.hh" -#endif #ifdef MODULE_PYTHON #include "PythonSegmentOrdering.hh" #endif @@ -215,10 +212,6 @@ const Core::ParameterBool CorpusDescription::paramSegmentOrderSortByTimeLength( const Core::ParameterInt CorpusDescription::paramSegmentOrderSortByTimeLengthChunkSize( "segment-order-sort-by-time-length-chunk-size", "Only sort each such chunk of segments. (-1 = disabled)", -1); -const Core::ParameterBool CorpusDescription::paramTheanoSegmentOrder( - "theano-segment-order", - "use theano to specify the order of segments over shared memory", - false); const Core::ParameterBool CorpusDescription::paramPythonSegmentOrder( "python-segment-order", "use Python to specify the order of segments", @@ -520,19 +513,10 @@ CorpusDescription::CorpusDescription(const Core::Configuration& c) // Handle the ordering. { - if (paramTheanoSegmentOrder(config)) { -#ifdef MODULE_THEANO_INTERFACE - verify(!ordering_); - ordering_ = new TheanoSegmentOrderingVisitor(); - log("Using Theano segment ordering"); -#else - criticalError("theano-segment-order not possible, MODULE_THEANO_INTERFACE disabled."); -#endif - } if (paramPythonSegmentOrder(config)) { #ifdef MODULE_PYTHON if (ordering_) - criticalError("python-segment-order not possible, another ordering (theano?) already used"); + criticalError("python-segment-order not possible, another ordering already used"); std::string pyModPath = paramPythonSegmentOrderModPath(config); std::string pyModName = paramPythonSegmentOrderModName(config); std::string pyConfig = paramPythonSegmentOrderConfig(config); diff --git a/src/Bliss/CorpusDescription.hh b/src/Bliss/CorpusDescription.hh index d83a8351b..6c853542c 100644 --- a/src/Bliss/CorpusDescription.hh +++ b/src/Bliss/CorpusDescription.hh @@ -590,7 +590,6 @@ public: static const Core::ParameterBool paramSegmentOrderSortByTimeLength; static const Core::ParameterInt paramSegmentOrderSortByTimeLengthChunkSize; static const Core::ParameterBool paramProgressReportingSegmentOrth; - static const Core::ParameterBool paramTheanoSegmentOrder; static const Core::ParameterBool paramPythonSegmentOrder; static const Core::ParameterString paramPythonSegmentOrderModPath; static const Core::ParameterString paramPythonSegmentOrderModName; diff --git a/src/Bliss/Makefile b/src/Bliss/Makefile index 5288decb8..d1fa7c98f 100644 --- a/src/Bliss/Makefile +++ b/src/Bliss/Makefile @@ -34,10 +34,6 @@ CHECK_O = $(OBJDIR)/check.o \ ../Math/libSprintMath.$(a) \ ../Core/libSprintCore.$(a) -ifdef MODULE_THEANO_INTERFACE - LIBBLISS_O += $(OBJDIR)/TheanoCommunicator.o - LIBBLISS_O += $(OBJDIR)/TheanoSegmentOrderingVisitor.o -endif ifdef MODULE_PYTHON LIBBLISS_O += $(OBJDIR)/PythonSegmentOrdering.o CHECK_O += ../Python/libSprintPython.$(a) diff --git a/src/Bliss/TheanoCommunicator.cc b/src/Bliss/TheanoCommunicator.cc deleted file mode 100644 index 8be206454..000000000 --- a/src/Bliss/TheanoCommunicator.cc +++ /dev/null @@ -1,201 +0,0 @@ -/** Copyright 2020 RWTH Aachen University. All rights reserved. - * - * Licensed under the RWTH ASR License (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.hltpr.rwth-aachen.de/rwth-asr/rwth-asr-license.html - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include - -#include "TheanoCommunicator.hh" - -using namespace Bliss; -using namespace std; - -namespace { -std::string getSegmentName(const Bliss::SpeechSegment* segment) { - return segment->recording()->parent()->name() + "/" + segment->recording()->name() + "/" + segment->name(); -} - -} // namespace - -const Core::ParameterInt TheanoCommunicator::paramSharedMemKey( - "shared-mem-key", "(random) number which is used as key for shared memory", -1); - -std::unique_ptr TheanoCommunicator::communicator_; - -TheanoCommunicator& TheanoCommunicator::communicator() { - hope(communicator_); - return *communicator_; -} - -enum TheanoStatus { - IDLE = 0, - REQUEST_ERRORSIGNAL = 1, - ERRORSIGNAL_WRITTEN = 2, - TERMINATED = 4 -}; - -namespace { -/** - * layout of shared memory: - * 4 bytes TheanoStatus flag - * 4 bytes float nRows - * 4 bytes float nCols - * CTL_SEG_SIZE-12 bytes segmentName (0 terminated) - * rest: data - **/ -const u32 SHARED_MEM_SIZE = 5 * 1024 * 1024; //5M should be enough -const u32 CTL_SEG_SIZE = 512; -const u32 STATUS_BEGIN = 0; -const u32 ROWS_BEGIN = 4; -const u32 COLS_BEGIN = 8; -const u32 LOSS_BEGIN = 12; -const u32 SEGMENT_NAME_BEGIN = 16; -const u32 DATA_BEGIN = CTL_SEG_SIZE; -const u32 MAX_SEG_NAME_LEN = DATA_BEGIN - SEGMENT_NAME_BEGIN; -} // namespace - -template -//careful: the offset is in bytes and not related to T -T& TheanoCommunicator::shMem(u32 offset) { - return *reinterpret_cast(static_cast(shMem_) + offset); -} - -u32& TheanoCommunicator::shMemStatus() { - return shMem(STATUS_BEGIN); -} - -float& TheanoCommunicator::shMemData(u32 idx) { - return shMem(DATA_BEGIN + 4 * idx); -} - -u32& TheanoCommunicator::shMemRows() { - return shMem(ROWS_BEGIN); -} - -u32& TheanoCommunicator::shMemCols() { - return shMem(COLS_BEGIN); -} - -float& TheanoCommunicator::shMemLoss() { - return shMem(LOSS_BEGIN); -} - -void TheanoCommunicator::waitForStatus(u32 status) { - while (!(shMemStatus() & status)) { - sched_yield(); - } -} - -TheanoCommunicator::TheanoCommunicator(const Core::Configuration& c) - : Core::Component(c) { - key_t shMemKey = paramSharedMemKey(c); - if (shMemKey == -1) { - criticalError("sharedMemKey not specified"); - } - log() << "allocating " << SHARED_MEM_SIZE << " bytes of shared memory..."; - shId_ = shmget(shMemKey, SHARED_MEM_SIZE, IPC_CREAT | 0660); - if (shId_ < 0) { - criticalError("failed to allocated shared memory (shmget failed)"); - } - shMem_ = shmat(shId_, 0, 0); - if (shMem_ == (void*)-1) { - criticalError("failed to allocate shared memory (shmat failed)"); - } - //shmctl(shId_, IPC_RMID, 0); //mark for deletion, will only be deleted after detaching - //note: seems not to work like this..., so do it in destructor instead - shMemStatus() = IDLE; - log() << "shared memory allocation was successfull"; -} - -TheanoCommunicator::~TheanoCommunicator() { - shmctl(shId_, IPC_RMID, 0); - shmdt(shMem_); -} - -void TheanoCommunicator::create(const Core::Configuration& c) { - if (!communicator_) { - communicator_.reset(new TheanoCommunicator(c)); - } -} - -bool TheanoCommunicator::waitForErrorSignalRequest(/*out*/ std::string& segmentName) { - waitForStatus(REQUEST_ERRORSIGNAL | TERMINATED); - if (shMemStatus() == TERMINATED) { - segmentName = "invalid"; - return false; - } - - segmentName = &shMem(SEGMENT_NAME_BEGIN); - log() << "error signal for segment " << segmentName << " was requested by theano"; - return true; -} - -const Math::Matrix& TheanoCommunicator::getPosteriorsForSegment(const Bliss::SpeechSegment* segment) { - std::string segmentName = getSegmentName(segment); - if (segmentName == currentSegmentName_) { - /* use caching only a single time per segment - assumption: each segment is requested 2 times in a row, - once for rescoring and once for accumulation - after this the same segment might be requested again - but then we want to get new posteriors from theano */ - currentSegmentName_ = ""; - return posteriors_; - } - - timeval start; - TIMER_START(start); - - //shared memory transmission - std::string shMemSegmentName = &shMem(SEGMENT_NAME_BEGIN); - if (segmentName != shMemSegmentName) { - criticalError("segment names do not match: ") << segmentName << " " << shMemSegmentName; - } - int rows = shMemRows(); - int cols = shMemCols(); - posteriors_.resize(rows, cols); - for (int i = 0; i < rows; ++i) { - for (int j = 0; j < cols; ++j) { - posteriors_[i][j] = shMemData(i * cols + j); - } - } - shMemStatus() = IDLE; - - currentSegmentName_ = segmentName; - timeval end; - f64 diff = 0; - TIMER_STOP(start, end, diff); - log() << "time getting posteriors: " << diff; - return posteriors_; -} - -void TheanoCommunicator::writeErrorSignalForSegment(const Bliss::SpeechSegment* segment, float loss, const Math::Matrix& errSig) { - std::string segmentName = getSegmentName(segment); - log() << "writing error signal for segment " << segmentName; - if (shMemStatus() != IDLE) { - criticalError("unexpected shMemStatus"); - } - shMemRows() = static_cast(errSig.nRows()); - shMemCols() = static_cast(errSig.nColumns()); - shMemLoss() = loss; - - for (u32 i = 0; i < errSig.nRows(); ++i) { - for (u32 j = 0; j < errSig.nColumns(); ++j) { - shMemData(i * errSig.nColumns() + j) = errSig[i][j]; - } - } - - shMemStatus() = ERRORSIGNAL_WRITTEN; - log() << "done writing error signal"; -} diff --git a/src/Bliss/TheanoCommunicator.hh b/src/Bliss/TheanoCommunicator.hh deleted file mode 100644 index 49af1b7dc..000000000 --- a/src/Bliss/TheanoCommunicator.hh +++ /dev/null @@ -1,63 +0,0 @@ -/** Copyright 2020 RWTH Aachen University. All rights reserved. - * - * Licensed under the RWTH ASR License (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.hltpr.rwth-aachen.de/rwth-asr/rwth-asr-license.html - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef THEANOCOMMUNICATOR_HH_ -#define THEANOCOMMUNICATOR_HH_ - -#include - -#include -#include - -#include "CorpusDescription.hh" - -namespace Bliss { - -/* - * single global instance -*/ -class TheanoCommunicator : public Core::Component { -private: - static const Core::ParameterInt paramSharedMemKey; - TheanoCommunicator(const Core::Configuration& c); - - //careful: the offset is in bytes and not related to T - template - T& shMem(u32 offset); - u32& shMemStatus(); - float& shMemData(u32 idx); //here idx is in floats, not in bytes! - u32& shMemRows(); - u32& shMemCols(); - float& shMemLoss(); - void waitForStatus(u32 status); - - static std::unique_ptr communicator_; - std::string currentSegmentName_; //used to cache posteriors for 1 segment - Math::Matrix posteriors_; - - int shId_; - void* shMem_; - -public: - ~TheanoCommunicator(); - static TheanoCommunicator& communicator(); - static void create(const Core::Configuration& c); - bool waitForErrorSignalRequest(/*out*/ std::string& segmentName); - const Math::Matrix& getPosteriorsForSegment(const Bliss::SpeechSegment* segment); - void writeErrorSignalForSegment(const Bliss::SpeechSegment* segment, float loss, const Math::Matrix& m_); -}; - -} // namespace Bliss - -#endif /*THEANOCOMMUNICATOR_HH_*/ diff --git a/src/Bliss/TheanoSegmentOrderingVisitor.cc b/src/Bliss/TheanoSegmentOrderingVisitor.cc deleted file mode 100644 index 309b3a0ce..000000000 --- a/src/Bliss/TheanoSegmentOrderingVisitor.cc +++ /dev/null @@ -1,55 +0,0 @@ -/** Copyright 2020 RWTH Aachen University. All rights reserved. - * - * Licensed under the RWTH ASR License (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.hltpr.rwth-aachen.de/rwth-asr/rwth-asr-license.html - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "TheanoSegmentOrderingVisitor.hh" - -#include - -#include "TheanoCommunicator.hh" - -using namespace Bliss; - -TheanoSegmentOrderingVisitor::~TheanoSegmentOrderingVisitor() { -} - -void TheanoSegmentOrderingVisitor::leaveCorpus(Bliss::Corpus* corpus) { - curCorpus_.pop_back(); - if (!curCorpus_.empty()) { - // not the root corpus - return; - } - - CustomCorpusGuide corpusGuide(this, /* root */ corpus); - std::string name; - while (TheanoCommunicator::communicator().waitForErrorSignalRequest(name)) - corpusGuide.showSegmentByName(name); -} - -SegmentOrderingVisitor* TheanoSegmentOrderingVisitor::copy() { - Core::Application::us()->error( - "TheanoSegmentOrderingVisitor: copy not supported (check progress-indication != global)"); - // copy() is currently only used by CorpusDescription::totalSegmentCount(). - // Maybe this could be implemented -- however, not sure if it's worth. - return 0; -} - -void TheanoSegmentOrderingVisitor::setAutoShuffle(bool enabled) { - if (enabled) - Core::Application::us()->error("TheanoSegmentOrderingVisitor: auto-shuffle not supported"); -} - -void TheanoSegmentOrderingVisitor::setSegmentList(const std::string& filename) { - Core::Application::us()->error( - "TheanoSegmentOrderingVisitor: segment list not supported: %s", filename.c_str()); -} diff --git a/src/Bliss/TheanoSegmentOrderingVisitor.hh b/src/Bliss/TheanoSegmentOrderingVisitor.hh deleted file mode 100644 index 45178d7d6..000000000 --- a/src/Bliss/TheanoSegmentOrderingVisitor.hh +++ /dev/null @@ -1,40 +0,0 @@ -/** Copyright 2020 RWTH Aachen University. All rights reserved. - * - * Licensed under the RWTH ASR License (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.hltpr.rwth-aachen.de/rwth-asr/rwth-asr-license.html - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef _BLISS_THEANO_SEGMENT_ORDERING_VISITOR_HH -#define _BLISS_THEANO_SEGMENT_ORDERING_VISITOR_HH - -#include "CorpusDescription.hh" -#include "SegmentOrdering.hh" - -namespace Bliss { - -/** - * Changes the order of processed segments according to the order given by Theano - * note that each segment can also be visited multiple times - */ -class TheanoSegmentOrderingVisitor : public Bliss::SegmentOrderingVisitor { -public: - virtual ~TheanoSegmentOrderingVisitor(); - virtual SegmentOrderingVisitor* copy(); - - virtual void leaveCorpus(Bliss::Corpus* corpus); - - virtual void setAutoShuffle(bool enabled); - virtual void setSegmentList(const std::string& filename); -}; - -} // namespace Bliss - -#endif // _BLISS_THEANO_SEGMENT_ORDERING_VISITOR_HH diff --git a/src/Speech/AcousticSegmentwiseTrainer.hh b/src/Speech/AcousticSegmentwiseTrainer.hh index fe912c50d..a6463d58c 100644 --- a/src/Speech/AcousticSegmentwiseTrainer.hh +++ b/src/Speech/AcousticSegmentwiseTrainer.hh @@ -37,8 +37,7 @@ class AbstractAcousticSegmentwiseTrainer : public AbstractSegmentwiseTrainer { public: enum ModelType { gaussianMixture, maximumEntropy, - neuralNetwork, - neuralNetworkTheano }; + neuralNetwork }; static Core::Choice choiceModelType; static Core::ParameterChoice paramModelType; diff --git a/src/Speech/LatticeSetExtractor.hh b/src/Speech/LatticeSetExtractor.hh index 0a849fb09..bb040bfed 100644 --- a/src/Speech/LatticeSetExtractor.hh +++ b/src/Speech/LatticeSetExtractor.hh @@ -45,7 +45,6 @@ protected: static const Core::ParameterStringVector paramAcousticExtractors; static const Core::ParameterStringVector paramEmissionExtractors; static const Core::ParameterStringVector paramNnEmissionExtractors; - static const Core::ParameterStringVector paramTheanoEmissionExtractors; static const Core::ParameterStringVector paramTdpExtractors; static const Core::ParameterStringVector paramPronunciationExtractors; static const Core::ParameterStringVector paramLmExtractors; @@ -102,7 +101,6 @@ private: void appendAcousticRescorers(); void appendEmissionRescorers(); void appendNnEmissionRescorers(); - void appendTheanoEmissionRescorers(); void appendTdpRescorers(); void appendPronunciationRescorers(); void appendLmRescorers();