Merge branch 'sprint_6' into 'hs-worms/TOP-2021'

[MERGE #145] sprint_6

See merge request top/21s/sivert-vis/sivert-vis-project!102

src/core/dataHandler/WDataSetCSV.h

@@ -60,6 +60,11 @@ public:
      */
     typedef std::vector< std::string > ContentElem;
 
+    /**
+     * represents a shared pointer to a ContentElem object.
+     */
+    typedef boost::shared_ptr< std::vector< std::string > > ContentElemSPtr;
+
     /**
      * Construct WDataSetCSV object
      *

src/modules/data/io/WReaderCSV.cpp

@@ -89,9 +89,6 @@ boost::shared_ptr< WDataSetCSV > WReaderCSV::read()
 
     file.close();
 
-    header->push_back( data->at( 0 ) );
-    data->erase( data->begin() );
-
     boost::shared_ptr< WDataSetCSV > datasetcsv = boost::shared_ptr< WDataSetCSV >( new WDataSetCSV( header, data ) );
     datasetcsv->setRawDataSet( rawRow );
 

src/modules/data/io/test/WReaderCSV_test.h

@@ -90,7 +90,42 @@ public:
         std::string fileName = W_FIXTURE_PATH + "CSVs/valid.csv";
         std::cout << std::endl << "Test loading of " << fileName << "." << std::endl;
 
-        TS_ASSERT_THROWS_NOTHING( WReaderCSV( fileName ).read() );
+
+        WDataSetCSV::ContentSPtr testHeader = WDataSetCSV::ContentSPtr( new WDataSetCSV::Content() );
+        WDataSetCSV::ContentSPtr testDataFirstRow = WDataSetCSV::ContentSPtr( new WDataSetCSV::Content() );
+        WDataSetCSV::ContentSPtr testDataLastRow = WDataSetCSV::ContentSPtr( new WDataSetCSV::Content() );
+        testHeader->push_back(
+            {
+                "PDGEncoding", "trackID", "parentID", "trackLocalTime", "time", "edep", "stepLength", "trackLength",
+                "posX", "posY", "posZ", "localPosX", "localPosY", "localPosZ", "momDirX", "momDirY", "momDirZ", "baseID",
+                "level1ID", "level2ID", "level3ID", "level4ID", "layerID", "photonID", "nPhantomCompton", "nCrystalCompton",
+                "nPhantomRayleigh", "nCrystalRayleigh", "primaryID", "sourcePosX", "sourcePosY", "sourcePosZ", "sourceID", "eventID",
+                "runID", "axialPos", "rotationAngle", "volumeID", "processName", "comptVolName", "RayleighVolName"
+            }
+        );
+        testDataFirstRow->push_back(
+            {
+                "2212", "1", "0", "4.20922e-09", "4.20922e-09", "0.0199097", "0.0250083", "725.314", "-1.86984", "4.55793", "225.244", "-1.86984",
+                "4.55793", "0.0125", "-0.00614681", "0.0255574", "0.999654", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "1", "-0.154322",
+                "0.156973", "-500", "0", "1", "0", "277.4", "0", "[0;1;0;3;-1;-1;-1;-1;-1;-1]", "Transportation", "NULL", "NULL"
+            }
+        );
+        testDataLastRow->push_back(
+            {
+                "2212", "1", "0", "5.92057e-09", "5.92057e-09", "0.3312", "0.025598", "955.947", "14.127", "28.6775", "454.394", "14.127", "28.6775",
+                "0.0125", "0.216802", "0.0338896", "0.975627", "1", "22", "0", "0", "0", "0", "0", "0", "0", "0", "0", "1", "-0.480459", "0.0634152",
+                "-500", "0", "2", "0", "461.8", "0", "[0;2;22;3;-1;-1;-1;-1;-1;-1]", "Transportation", "NULL", "NULL"
+            }
+        );
+
+        WReaderCSV tmpCsvReader( fileName );
+        TS_ASSERT_THROWS_NOTHING( tmpCsvReader.read() );
+        // compare headers
+        TS_ASSERT_EQUALS( tmpCsvReader.read()->getHeader()->front(), testHeader->front() );
+        // compare first data rows
+        TS_ASSERT_EQUALS( tmpCsvReader.read()->getData()->front(), testDataFirstRow->front() );
+        // compare last data rows
+        TS_ASSERT_EQUALS( tmpCsvReader.read()->getData()->back(), testDataLastRow->front() );
     }
 };
 

src/modules/filterProtonData/WCsvConverter.cpp

@@ -22,6 +22,7 @@
 //
 //---------------------------------------------------------------------------
 
+#include <cmath>
 #include <string>
 #include <vector>
 
@@ -76,6 +77,7 @@ void WCsvConverter::setOutputFromCSV( )
     m_indexes->update( m_protonData );
 
     float maxEdep = 0.0;
+    float minEdep = 1.0;
 
     for( WDataSetCSV::Content::iterator dataRow = data->begin(); dataRow < data->end(); dataRow++ )
     {
@@ -84,9 +86,19 @@ void WCsvConverter::setOutputFromCSV( )
             continue;
         }
 
+        if( m_protonData->isColumnAvailable( WSingleSelectorName::getEdep() ) )
+        {
+            float edep = stringToDouble( dataRow->at( m_indexes->getEdep() ) );
+
+            if( getClusterSize( edep ) < 1.0 || getClusterSize( edep ) > 35.0 )
+            {
+                continue;
+            }
+        }
+
         addVertex( dataRow );
         addColor( plainColor );
-        addEdepAndSize( dataRow, &maxEdep );
+        addEdepAndSize( dataRow, &maxEdep, &minEdep );
         addEventID( dataRow );
     }
 
@@ -95,7 +107,7 @@ void WCsvConverter::setOutputFromCSV( )
         return;
     }
 
-    normalizeEdeps( m_vectors->getEdeps(), m_vectors->getColors(),  maxEdep );
+    normalizeEdeps( m_vectors->getEdeps(), m_vectors->getColors(),  maxEdep, minEdep );
 
     createOutputPoints();
     createOutputFibers();
@@ -109,16 +121,16 @@ boost::shared_ptr< WDataSetSingle > WCsvConverter::getTransferFunction()
 
 boost::shared_ptr< std::vector<unsigned char> > WCsvConverter::sampleTransferFunction()
 {
-    boost::shared_ptr< std::vector<unsigned char> > data( new std::vector<unsigned char>( 10 * 4 ) );
+    boost::shared_ptr< std::vector<unsigned char> > data( new std::vector<unsigned char>( 50 * 4 ) );
 
     WTransferFunction tf = m_propertyStatus->getVisualizationPropertyHandler()->getTransferFunction()->get( true );
 
-    tf.sample1DTransferFunction( &( *data )[ 0 ], 10, 0.0, 1.0 );
+    tf.sample1DTransferFunction( &( *data )[ 0 ], 50, 0.0, 1.0 );
 
     return data;
 }
 
-void WCsvConverter::normalizeEdeps( SPFloatVector edeps, SPFloatVector colorArray, float maxEdep )
+void WCsvConverter::normalizeEdeps( SPFloatVector edeps, SPFloatVector colorArray, float maxEdep, float minEdep )
 {
     if( m_protonData->isColumnAvailable( WSingleSelectorName::getEdep() ) )
     {
@@ -127,6 +139,7 @@ void WCsvConverter::normalizeEdeps( SPFloatVector edeps, SPFloatVector colorArra
         setTransferFunction( data );
 
         float maxClusterSize = getClusterSize( maxEdep );
+        float minClusterSize = getClusterSize( minEdep );
 
         for( std::vector< float >::iterator currentEdep = edeps->begin();
             currentEdep != edeps->end();
@@ -138,7 +151,7 @@ void WCsvConverter::normalizeEdeps( SPFloatVector edeps, SPFloatVector colorArra
 
             if( m_propertyStatus->getVisualizationPropertyHandler()->getColorFromEdep()->get() )
             {
-                clusterSizeNormalized = static_cast< int >( 9 * clusterSizeNormalized );
+                clusterSizeNormalized = static_cast< int >( 49 * clusterSizeNormalized );
 
                 for( int i = 0; i < 4; i++ )
                 {
@@ -149,6 +162,7 @@ void WCsvConverter::normalizeEdeps( SPFloatVector edeps, SPFloatVector colorArra
 
         m_colorBar->getProperties()->getProperty( "Max scale value" )->set( 0.0 );
         m_colorBar->getProperties()->getProperty( "Max scale value" )->set( maxClusterSize );
+        m_colorBar->getProperties()->getProperty( "Min scale value" )->set( minClusterSize );
         m_colorBar->getProperties()->getProperty( "Description" )->set( std::string( "Clustersize " ) );
 
         bool activated = m_propertyStatus->getVisualizationPropertyHandler()->getColorFromEdep()->get();
@@ -224,7 +238,7 @@ void WCsvConverter::addColor( WColor plainColor )
     }
 }
 
-void WCsvConverter::addEdepAndSize( WDataSetCSV::Content::iterator dataRow, float* maxEdep )
+void WCsvConverter::addEdepAndSize( WDataSetCSV::Content::iterator dataRow, float* maxEdep, float* minEdep )
 {
     if( !m_protonData->isColumnAvailable( WSingleSelectorName::getEdep() ) )
     {
@@ -237,6 +251,11 @@ void WCsvConverter::addEdepAndSize( WDataSetCSV::Content::iterator dataRow, floa
         *maxEdep = edep;
     }
 
+    if( edep < *minEdep )
+    {
+        *minEdep = edep;
+    }
+
     m_vectors->getEdeps()->push_back( edep );
 }
 
@@ -371,7 +390,7 @@ void WCsvConverter::setTransferFunction( boost::shared_ptr< std::vector<unsigned
     boost::shared_ptr< WValueSetBase > newValueSet( new WValueSet<unsigned char>( 1, 4, data, W_DT_UNSIGNED_CHAR ) );
 
     WGridTransformOrtho transform;
-    boost::shared_ptr< WGridRegular3D > newGrid( new WGridRegular3D( 10, 1, 1, transform ) );
+    boost::shared_ptr< WGridRegular3D > newGrid( new WGridRegular3D( 50, 1, 1, transform ) );
     boost::shared_ptr< WDataSetSingle > newData( new WDataSetSingle( newValueSet, newGrid ) );
 
     m_transferFunction = newData;
@@ -391,7 +410,7 @@ bool WCsvConverter::checkIfOutputIsNull()
 
 float WCsvConverter::getClusterSize( float edep )
 {
-    return 2.4 * pow( edep, 0.338 );
+    return 7.6626f * powf( edep * 40.0f, 0.420307f );
 }
 
 float WCsvConverter::stringToDouble( std::string str )

src/modules/filterProtonData/WCsvConverter.h

@@ -53,6 +53,10 @@
  */
 class WCsvConverter
 {
+    /**
+     * only test classes may be friend
+     */
+     friend class WCsvConverterTest;
 public:
     /**
      * Initializes the vectors, indices and transfer function color bar
@@ -156,8 +160,9 @@ private:
      * \param edeps vector containing energy deposition values
      * \param colorArray vector containing colors per vertex
      * \param maxEdep maximum energy deposition
+     * \param minEdep minimum energy deposition
      */
-    void normalizeEdeps( SPFloatVector edeps, SPFloatVector colorArray,  float maxEdep );
+    void normalizeEdeps( SPFloatVector edeps, SPFloatVector colorArray,  float maxEdep, float minEdep );
 
 
     /**
@@ -187,8 +192,9 @@ private:
      *
      * \param dataRow each row of the csv file (content of row)
      * \param maxEdep a pointer to the current maximum of the edep
+     * \param minEdep a pointer to the current minimum of the edep
      */
-    void addEdepAndSize( WDataSetCSV::Content::iterator dataRow, float* maxEdep );
+    void addEdepAndSize( WDataSetCSV::Content::iterator dataRow, float* maxEdep, float* minEdep );
 
     /**
      * Create eventID for Fiber renderer

src/modules/filterProtonData/WDataType.cpp

+//---------------------------------------------------------------------------
+//
+// Project: OpenWalnut ( http://www.openwalnut.org )
+//
+// Copyright 2009 OpenWalnut Community, BSV@Uni-Leipzig and CNCF@MPI-CBS
+// For more information see http://www.openwalnut.org/copying
+//
+// This file is part of OpenWalnut.
+//
+// OpenWalnut is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// OpenWalnut is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with OpenWalnut. If not, see <http://www.gnu.org/licenses/>.
+//
+//---------------------------------------------------------------------------
+
+#include <string>
+
+#include "WDataType.h"
+
+std::string WDataType::getInt()
+{
+    return "int";
+}
+
+std::string WDataType::getDouble()
+{
+    return "double";
+}
+
+std::string WDataType::getString()
+{
+    return "string";
+}
+
+std::string WDataType::getDefault()
+{
+    return "default";
+}

src/modules/filterProtonData/WDataType.h

+//---------------------------------------------------------------------------
+//
+// Project: OpenWalnut ( http://www.openwalnut.org )
+//
+// Copyright 2009 OpenWalnut Community, BSV@Uni-Leipzig and CNCF@MPI-CBS
+// For more information see http://www.openwalnut.org/copying
+//
+// This file is part of OpenWalnut.
+//
+// OpenWalnut is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// OpenWalnut is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with OpenWalnut. If not, see <http://www.gnu.org/licenses/>.
+//
+//---------------------------------------------------------------------------
+
+#ifndef WDATATYPE_H
+#define WDATATYPE_H
+
+#include <string>
+
+/**
+ * Holds the datatypes as string
+ */
+class WDataType
+{
+public:
+    /**
+     * getter
+     * \return datatype int as string
+     */
+    static std::string getInt();
+
+    /**
+     * getter
+     * \return datatype double as string
+     */
+    static std::string getDouble();
+
+    /**
+     * getter
+     * \return datatype string as string
+     */
+    static std::string getString();
+
+    /**
+     * getter
+     * \return default as string
+     */
+    static std::string getDefault();
+};
+
+#endif  // WDATATYPE_H

src/modules/filterProtonData/WProtonData.cpp

@@ -22,8 +22,11 @@
 //
 //---------------------------------------------------------------------------
 
+#include <regex>
+#include <list>
 #include <string>
 #include <vector>
+#include <boost/lexical_cast.hpp>
 
 #include "WProtonData.h"
 
@@ -66,7 +69,14 @@ void WProtonData::setCSVData( WDataSetCSV::ContentSPtr csvData )
         throw WException( "Can not set data! No data specified!" );
     }
 
+    if( csvData->empty() )
+    {
+        throw WException( "Can not set data! No data content found!" );
+    }
+
     m_csvData = csvData;
+
+    detectColumnTypesFromCsvData( csvData );
 }
 
 WDataSetCSV::ContentSPtr WProtonData::getCSVData()
@@ -105,3 +115,99 @@ int WProtonData::getColumnIndex( std::string columnName )
 
     return m_columnMap[ columnName ];
 }
+
+WDataSetCSV::ContentElemSPtr WProtonData::getColumnTypes()
+{
+    return m_columnTypes;
+}
+
+void WProtonData::detectColumnTypesFromCsvData( WDataSetCSV::ContentSPtr csvData )
+{
+    m_columnTypes = WDataSetCSV::ContentElemSPtr( new std::vector< std::string >() );
+
+    auto currentRow = csvData->begin();
+
+    // determine column types based on first csv data row
+    for( auto cell : *currentRow )
+    {
+        m_columnTypes->push_back( determineColumnTypeByString( cell ) );
+    }
+
+    assert( m_columnTypes != nullptr );
+    assert( !m_columnTypes->empty() );
+    assert( m_columnTypes->size() == m_csvHeader->at( 0 ).size() );
+
+    for( size_t idx = 0; idx < m_columnTypes->size(); idx++ )
+    {
+        if(m_columnTypes->at( idx ) == WDataType::getDouble() )
+        {
+            if( checkIfDoubleColumnCanBeInteger( idx ) )
+            {
+                m_columnTypes->at( idx ) = WDataType::getInt();
+            }
+        }
+    }
+}
+
+std::string WProtonData::determineColumnTypeByString( std::string cellValue )
+{
+    std::regex regexInt( R"(^[-\+]?[[:d:]]+([eE]\+?0?[1-9])?$)" );
+    std::regex regexDouble( R"(^([+-]?(?:[[:d:]]+\.?|[[:d:]]*\.[[:d:]]+))(?:[Ee][+-]?[[:d:]]+)?$)" );
+
+    if( std::regex_search( cellValue, regexInt ) )
+    {
+        return WDataType::getInt();
+    }
+    else if( std::regex_search( cellValue, regexDouble ) )
+    {
+        return WDataType::getDouble();
+    }
+    else
+    {
+        return WDataType::getString();
+    }
+}
+
+bool WProtonData::checkIfDoubleColumnCanBeInteger( int columnNumber )
+{
+    double doubleValue;
+    int intValue;
+
+    for( auto row : *m_csvData )
+    {
+        doubleValue = boost::lexical_cast< double >( row.at( columnNumber ) );
+        intValue = ( int )doubleValue;
+
+        if( doubleValue - intValue != 0 )
+        {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+std::vector< std::string > WProtonData::getHeaderFromType( std::list< std::string > typeNames )
+{
+    std::vector< std::string > header = m_csvHeader->at( 0 );
+    std::vector< std::string > columnTypes = *m_columnTypes;
+    std::vector< std::string > filterHeader;
+
+    for( size_t i = 0; i < columnTypes.size(); i++)
+    {
+        for( std::string type : typeNames )
+        {
+            if( type == WDataType::getDefault() )
+            {
+                return header;
+            }
+
+            if( columnTypes[ i ] == type )
+            {
+                filterHeader.push_back( header[i] );
+            }
+        }
+    }
+
+    return filterHeader;
+}

src/modules/filterProtonData/WProtonData.h

@@ -25,10 +25,14 @@
 #ifndef WPROTONDATA_H
 #define WPROTONDATA_H
 
+#include <regex>
+#include <list>
 #include <map>
 #include <string>
+#include <vector>
 
 #include "core/dataHandler/WDataSetCSV.h"
+#include "WDataType.h"
 
 
 /**
@@ -113,6 +117,20 @@ public:
      */
     bool isColumnAvailable( std::string columnName );
 
+    /**
+     * Get column types, stored in a string vector.
+     * Positions within this vector are linked to positions in m_csvHeader
+     * \return a shared pointer to m_columnTypes
+     */
+    WDataSetCSV::ContentElemSPtr getColumnTypes();
+
+    /**
+     * Return a vector of filtered Headers
+     * \param typeNames Types of filter 
+     * \return Return a vector of filtered Headers
+     */
+    std::vector< std::string > getHeaderFromType( std::list< std::string > typeNames );
+
 private:
     /**
      * Stores column index of data.
@@ -133,6 +151,31 @@ private:
      * Stores index of the selected single-selector (ColumnPropertyHandler)
      */
     std::map< std::string, int > m_ColumnMapSelectedIndex;
+
+    /**
+     * Stores the information, which data type is stored in associated column
+     */
+    WDataSetCSV::ContentElemSPtr m_columnTypes;
+
+    /**
+     * Reads csv data and stores column types in m_columnTypes
+     * \param csvData the input csv data
+     */
+    void detectColumnTypesFromCsvData( WDataSetCSV::ContentSPtr csvData );
+
+    /**
+     * Determines column type due to cellValue
+     * \param cellValue the value of a cell on the basis of which the column type is to be determined
+     * \return either "int", "double" or "string"
+     */
+    std::string determineColumnTypeByString( std::string cellValue );
+
+    /**
+     * Checks, if values of a column, containing double values, can be converted to integers
+     * \param columnNumber the column number within m_csvHeader
+     * \return true, if all double values of a column ends with ".0"; false otherwise
+     */
+    bool checkIfDoubleColumnCanBeInteger( int columnNumber );
 };
 
 #endif  // WPROTONDATA_H

src/modules/filterProtonData/WSingleSelectorName.cpp

@@ -25,6 +25,7 @@
 #include <list>
 #include <string>
 
+#include "WDataType.h"
 #include "WSingleSelectorName.h"
 
 std::string WSingleSelectorName::getX()
@@ -62,61 +63,96 @@ std::string WSingleSelectorName::getParentId()
     return "Parent id";
 }
 
-WSingleSelectorName::NameDescriptionSearch WSingleSelectorName::getXwithDescription()
+WSingleSelectorName::NameDescriptionSearchTyp WSingleSelectorName::getXwithDescription()
 {
-    return WSingleSelectorName::NameDescriptionSearch( getX(),
-                        "Choose the column which should be used to determine the x coordinate.",
-                        "posX" );
+    return WSingleSelectorName::NameDescriptionSearchTyp(
+            getX(),
+            "Choose the column which should be used to determine the x coordinate.",
+            "posX",
+            std::list< std::string >
+                    {
+                        WDataType::getInt(), WDataType::getDouble()
+                    } );
 }
 
-WSingleSelectorName::NameDescriptionSearch WSingleSelectorName::getYwithDescription()
+WSingleSelectorName::NameDescriptionSearchTyp WSingleSelectorName::getYwithDescription()
 {
-    return WSingleSelectorName::NameDescriptionSearch( getY(),
-                        "Choose the column which should be used to determine the y coordinate.",
-                        "posY" );
+    return WSingleSelectorName::NameDescriptionSearchTyp(
+            getY(),
+            "Choose the column which should be used to determine the y coordinate.",
+            "posY",
+            std::list< std::string >
+                    {
+                        WDataType::getInt(), WDataType::getDouble()
+                    } );
 }
 
-WSingleSelectorName::NameDescriptionSearch WSingleSelectorName::getZwithDescription()
+WSingleSelectorName::NameDescriptionSearchTyp WSingleSelectorName::getZwithDescription()
 {
-    return WSingleSelectorName::NameDescriptionSearch( getZ(),
-                        "Choose the column which should be used to determine the z coordinate.",
-                        "posZ" );
+    return WSingleSelectorName::NameDescriptionSearchTyp(
+            getZ(),
+            "Choose the column which should be used to determine the z coordinate.",
+            "posZ",
+            std::list< std::string >
+                    {
+                        WDataType::getInt(), WDataType::getDouble()
+                    } );
 }
 
-WSingleSelectorName::NameDescriptionSearch WSingleSelectorName::getPDGwithDescription()
+WSingleSelectorName::NameDescriptionSearchTyp WSingleSelectorName::getPDGwithDescription()
 {
-    return WSingleSelectorName::NameDescriptionSearch( getPDG(),
-                        "Choose the column which should be used to determine the particle data group.",
-                        "PDGEncoding" );
+    return WSingleSelectorName::NameDescriptionSearchTyp(
+            getPDG(),
+            "Choose the column which should be used to determine the particle data group.",
+            "PDGEncoding",
+            std::list< std::string >
+                    {
+                        WDataType::getInt()