TableParseQuery.h

Go to the documentation of this file.

// # TableParseQuery.h: Class getting the parser results and executing a query
// # Copyright (C) 1994-2022
// # Associated Universities, Inc. Washington DC, USA.
// #
// # This library is free software; you can redistribute it and/or modify it
// # under the terms of the GNU Library General Public License as published by
// # the Free Software Foundation; either version 2 of the License, or (at your
// # option) any later version.
// #
// # This library is distributed in the hope that it will be useful, but WITHOUT
// # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
// # License for more details.
// #
// # You should have received a copy of the GNU Library General Public License
// # along with this library; if not, write to the Free Software Foundation,
// # Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
// #
// # Correspondence concerning AIPS++ should be addressed as follows:
// #        Internet email: casa-feedback@nrao.edu.
// #        Postal address: AIPS++ Project Office
// #                        National Radio Astronomy Observatory
// #                        520 Edgemont Road
// #                        Charlottesville, VA 22903-2475 USA
 
#ifndef TABLES_TABLEPARSEQUERY_H
#define TABLES_TABLEPARSEQUERY_H
 
// # Includes
#include <casacore/casa/aips.h>
#include <casacore/tables/TaQL/TableParseTableList.h>
#include <casacore/tables/TaQL/TableParseJoin.h>
#include <casacore/tables/TaQL/TableParseProject.h>
#include <casacore/tables/TaQL/TableParseUpdate.h>
#include <casacore/tables/TaQL/TableParseSortKey.h>
#include <casacore/tables/TaQL/TableParseGroupby.h>
#include <casacore/tables/Tables/Table.h>
#include <casacore/tables/TaQL/ExprNode.h>
#include <casacore/tables/TaQL/ExprGroup.h>
#include <casacore/casa/Arrays/ArrayFwd.h>
#include <casacore/casa/BasicSL/String.h>
#include <casacore/casa/Utilities/Sort.h>
#include <casacore/casa/Containers/Block.h>
#include <vector>
 
namespace casacore {  // # NAMESPACE CASACORE - BEGIN
 
// # Forward Declarations
class TableExprNodeSet;
class TableExprNodeSetElem;
class TableExprNodeIndex;
class TableDesc;
class TableColumn;
class AipsIO;
class Record;
class TableRecord;
template <class T>
class ArrayColumn;
 
// <summary>
// Class getting the parser results and executing a query
// </summary>
 
// <use visibility=local>
 
// <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
// </reviewed>
 
// <prerequisite>
// # Classes you should understand before using this one.
//  <li> TableGram.ll and .yy  (flex and bison grammar)
//  <li> TaQLNodeHandler
// </prerequisite>
 
// <synopsis>
// The results of the bison parser TableGram.yy and flex scanner TableGram.ll
// are stored in a tree of TaQLNode objects. When the parsing of a TaQL command
// is fully done, the tree is traversed by TaQLNodeHandler which creates and
// fills a stack of TableParseQuery objects, one object per (nested) query.
// A nested query is executed once it is fully handled.
// </synopsis>
 
// <motivation>
// It is necessary to be able to give a table select command in ASCII.
// This can be used in a CLI or in the table browser to get a subset
// of a table or to sort a table.
// </motivation>
 
// # <todo asof="$DATE:$">
// # A List of bugs, limitations, extensions or planned refinements.
// # </todo>
 
class TableParseQuery {
 public:
  enum CommandType {
    PSELECT,
    PUPDATE,
    PINSERT,
    PDELETE,
    PCOUNT,
    PCALC,
    PCRETAB,
    PALTTAB,
    PDROPTAB,
    PSHOW
  };
 
  // Construct.
  TableParseQuery(CommandType type);
 
  // Destructor.
  ~TableParseQuery();
 
  // Return the command type.
  CommandType commandType() const { return commandType_p; }
 
  // Return the expression node.
  TableExprNode getNode() const { return node_p; }
 
  // Return the table name list.
  TableParseTableList& tableList() { return tableList_p; }
 
  Table& projectExprTable() { return projectExprTable_p; }
 
  // Execute the select command (select/sort/projection/groupby/having/giving).
  // The setInGiving flag tells if a set in the GIVING part is allowed.
  // The mustSelect flag tells if a SELECT command must do something.
  // Usually that is required, but not for a SELECT in an INSERT command.
  // Optionally the maximum nr of rows to be selected can be given.
  // It will be used as the default value for the LIMIT clause.
  // 0 = no maximum.
  void execute(Bool showTimings, Bool setInGiving, Bool mustSelect, rownr_t maxRow,
               Bool doTracing = False,
               const std::vector<const Table*>& tempTables = std::vector<const Table*>(),
               const std::vector<TableParseQuery*>& stack = std::vector<TableParseQuery*>());
 
  // Execute a query in a FROM clause resulting in a Table.
  Table doFromQuery(Bool showTimings);
 
  // Execute a subquery and create an appropriate node for the result.
  TableExprNode doSubQuery(Bool showTimings);
 
  // Test if a subquery has sufficient elements.
  // It uses default LIMIT=1, but that can be overidden in the subquery.
  // The flag tells if NOT EXISTS or EXISTS was given.
  TableExprNode doExists(Bool noexists, Bool showTimings);
 
  // Show the expression tree.
  void show(ostream& os) const;
 
  // Create a temporary table if no tables are given in FROM.
  void handleTableNoFrom();
 
  // Keep the selection expression.
  void handleWhere(const TableExprNode&);
 
  // Keep the groupby expressions.
  // It checks if they are all scalar expressions.
  void handleGroupby(const std::vector<TableExprNode>&, Bool rollup);
 
  // Keep the having expression.
  void handleHaving(const TableExprNode&);
 
  // Keep the expression of a calculate command.
  void handleCalcComm(const TableExprNode&);
 
  // Handle the DROP TABLE command.
  void handleDropTab(const std::vector<const Table*>& tempTables,
                     const std::vector<TableParseQuery*>& stack);
 
  // Keep the create table command.
  void handleCreTab(const Record& dmInfo, const std::vector<const Table*>& tempTables,
                    const std::vector<TableParseQuery*>& stack);
 
  // Keep the column specification in a create table command.
  void handleColSpec(const String& columnName, const String& likeColName, const String& dataType,
                     const Record& spec, Bool isCOrder = False);
 
  // Reopen the table (for update) used in the ALTER TABLE command.
  void handleAltTab();
 
  // Add columns to the table of ALTER TABLE.
  // The column descriptions have already been added to tableDesc_p.
  void handleAddCol(const Record& dmInfo);
 
  // Handle copying of columns.
  void handleCopyCol(Bool showTimings);
 
  // Add a keyword or replace a keyword with a value.
  // The keyword can be a table or column keyword (col::key).
  // The data type string can be empty leaving the data type unchanged.
  void handleSetKey(const String& name, const String& dtype, const ValueHolder& value);
 
  // Rename a table or column keyword.
  void handleRenameKey(const String& oldName, const String& newName);
 
  // Remove a table or column keyword.
  void handleRemoveKey(const String& name);
 
  // Keep the update expressions.
  void handleUpdate();
 
  // Make ready for the insert expression.
  // The first one uses values (added via addUpdate),
  // the second one a subquery.
  // <group>
  void handleInsert();
  void handleInsert(TableParseQuery* sel);
  // </group>
 
  // Make ready for a COUNT command.
  // It checks if all column expressions are scalar.
  void handleCount();
 
  // Keep the sort expressions.
  void handleSort(const std::vector<TableParseSortKey>& sortList, Bool noDuplicates,
                  Sort::Order defaultSortOrder);
 
  // Evaluate and keep limit/offset/stride given as start:end:incr
  void handleLimit(const TableExprNodeSetElem& expr);
 
  // Evaluate and keep the limit value.
  void handleLimit(const TableExprNode& expr);
 
  // Evaluate and keep the offset value.
  void handleOffset(const TableExprNode& expr);
 
  // Evaluate and add the rows.
  void handleAddRow(const TableExprNode& expr);
 
  // Add a join object.
  TableParseJoin& addJoin();
 
  // Find the keyword or column name and create a TableExprNode from it.
  // If <src>tryProj=True</src> it is first tried if the column is a column
  // in the projected table (i.e., result from the SELECT part).
  TableExprNode handleKeyCol(const String& name, Bool tryProj);
 
  // Handle a slice operator.
  static TableExprNode handleSlice(const TableExprNode& array, const TableExprNodeSet& indices,
                                   const TaQLStyle&);
 
  // Handle a function.
  TableExprNode handleFunc(const String& name, const TableExprNodeSet& arguments, const TaQLStyle&);
 
  // Add a column to the list of column names.
  void handleColumn(Int type, const String& name, const TableExprNode& expr, const String& newName,
                    const String& nameMask, const String& newDtype);
 
  // Finish the addition of columns to the list of column names.
  void handleColumnFinish(Bool distinct);
 
  // Handle the name and type given in a GIVING clause.
  void handleGiving(const String& name, const Record& type);
 
  // Handle the set given in a GIVING clause.
  void handleGiving(const TableExprNodeSet&);
 
  // Get the TableParseJoin objects.
  const std::vector<TableParseJoin>& joins() const { return joins_p; }
 
  // Initialize the table and data manager descriptions.
  void initDescriptions(const TableDesc&, const Record& dminfo);
 
  // Add a keyword or replace a keyword with the value of another keyword.
  // The keywords can be table or column keywords (col::key).
  ValueHolder getRecFld(const String& name);
 
  // Split the given name into optional shorthand, column and fields.
  // Find the keywordset for it and fill in the final keyword name.
  // It is a helper function for handleSetKey, etc.
  // If update=True, rwKeywordSet() is used to ensure the table is updated.
  TableRecord& findKeyword(const String& name, String& keyName, Bool update = True);
 
  // Add an update object.
  void addUpdate(const std::shared_ptr<TableParseUpdate>& upd) { update_p.push_back(upd); }
 
  // Set the insert expressions for all rows.
  void setInsertExprs(const std::vector<TableExprNode> exprs) { insertExprs_p = exprs; }
 
  // Replace the first table (used by CALC command).
  void replaceTable(const Table& table);
 
  // Set the DataManager info for a new table.
  void setDMInfo(const Record& dminfo) { tableProject_p.setDMInfo(dminfo); }
 
  // Get the projected column names.
  const Block<String>& getColumnNames() const { return tableProject_p.getColumnNames(); }
 
  // Get the resulting table.
  const Table& getTable() const { return table_p; }
 
  // Show the structure of fromTables_p[0] using the options given in parts[2:].
  String getTableStructure(const Vector<String>& parts, const TaQLStyle& style);
 
  // Add a column node to applySelNodes_p.
  void addApplySelNode(const TableExprNode& node) { applySelNodes_p.push_back(node); }
 
  // Create a table using the given parameters.
  // The variables set by handleGiven are used for name and type.
  Table createTable(const TableDesc& td, Int64 nrow, const Record& dmInfo,
                    const std::vector<const Table*>& tempTables,
                    const std::vector<TableParseQuery*>& stack);
 
 private:
  // Do the update step.
  // Rows 0,1,2,.. in UpdTable are updated from the expression result
  // for the rows in the given rownrs vector.
  void doUpdate(Bool showTimings, const Table& origTable, Table& updTable,
                const Vector<rownr_t>& rownrs,
                const std::shared_ptr<TableExprGroupResult>& groups =
                    std::shared_ptr<TableExprGroupResult>());
 
  // Do the insert step and return a selection containing the new rows.
  Table doInsert(Bool showTimings, Table& table);
 
  // Do the delete step.
  void doDelete(Bool showTimings, Table& table);
 
  // Do the count step returning a memory table containing the unique
  // column values and the counts of the column values.
  Table doCount(Bool showTimings, const Table&);
 
  // Do the projection step returning a table containing the projection.
  Table doProject(Bool showTimings, const Table&,
                  const std::shared_ptr<TableExprGroupResult>& groups =
                      std::shared_ptr<TableExprGroupResult>());
 
  // Do the projection containing column expressions.
  // Use the selected or unselected columns depending on <src>useSel</src>.
  Table doProjectExpr(Bool useSel, const std::shared_ptr<TableExprGroupResult>& groups);
 
  // Create a subtable (used by createTable).
  Table createSubTable(const String& subtableName, const TableDesc& td, Int64 nrow,
                       const Record& dmInfo, const std::vector<const Table*>& tempTables,
                       const std::vector<TableParseQuery*>& stack);
 
  // Set the selected rows for the column objects in applySelNodes_p.
  // These nodes refer the original table. They requires different row
  // numbers than the selected groups and projected columns.
  // rownrs_p is changed to use row 0..n.
  // It returns the Table containing the subset of rows in the input Table.
  Table adjustApplySelNodes(const Table&);
 
  // Do the groupby/aggregate step and return its result.
  std::shared_ptr<TableExprGroupResult> doGroupby(bool showTimings);
 
  // Do the HAVING step.
  // It returns False if no HAVING step was given.
  Bool doHaving(Bool showTimings, const std::shared_ptr<TableExprGroupResult>& groups);
 
  // Do the sort step.
  void doSort(Bool showTimings);
 
  // Do the limit/offset step.
  void doLimOff(Bool showTimings);
  Table doLimOff(Bool showTimings, const Table& table);
 
  // Do the 'select distinct' step.
  Table doDistinct(Bool showTimings, const Table& table);
 
  // Finish the table (rename, copy, and/or flush).
  Table doFinish(Bool showTimings, Table& table, const std::vector<const Table*>& tempTables,
                 const std::vector<TableParseQuery*>& stack);
 
  // Make an array from the contents of a column in a subquery.
  TableExprNode getColSet();
 
  // Make a set from the results of the subquery.
  TableExprNode makeSubSet() const;
 
  // Evaluate an int scalar expression.
  Int64 evalIntScaExpr(const TableExprNode& expr) const;
 
  // # Data mambers.
  // # Command type.
  CommandType commandType_p;
  // # List of TableParsePair objects (from WITH and FROM clause).
  TableParseTableList tableList_p;
  // # A join object per join clause.
  std::vector<TableParseJoin> joins_p;
  // # Object holding the info of table projection (i.e., column selection).
  TableParseProject tableProject_p;
  // # Name and type of the resulting table (from GIVING part).
  String resultName_p;
  uInt resultType_p;     // # 0-unknown 1=memory 2=scratch 3=plain
  Bool resultCreated_p;  // # Has the result table been created?
  StorageOption storageOption_p;
  Table::EndianFormat endianFormat_p;
  Bool overwrite_p;
  // # Resulting set (from GIVING part).
  TableExprNodeSet* resultSet_p;
  // # The WHERE expression tree.
  TableExprNode node_p;
  // # The GROUPBY, aggregate and HAVING info.
  TableParseGroupby groupby_p;
  // # Distinct values in output?
  Bool distinct_p;
  // # The possible limit (= max nr of selected rows) (0 means no limit).
  Int64 limit_p;
  // # The possible last row (0 means no end; can be <0).
  // # limit_p and endrow_p cannot be both !=0.
  Int64 endrow_p;
  // # The possible offset (= nr of selected rows to skip).
  Int64 offset_p;
  // # The possible stride in offset:endrow:stride.
  Int64 stride_p;
  // # The update and insert list.
  std::vector<std::shared_ptr<TableParseUpdate>> update_p;
  // # The insert expressions (possibly for multiple rows).
  std::vector<TableExprNode> insertExprs_p;
  // # The table selection to be inserted.
  TableParseQuery* insSel_p;
  // # The sort list.
  std::vector<TableParseSortKey> sort_p;
  // # The noDuplicates sort switch.
  Bool noDupl_p;
  // # The default sort order.
  Sort::Order order_p;
  // # All nodes that need to be adjusted for a selection of rownrs.
  // # It can consist of column nodes and the rowid function node.
  // # Some nodes (in aggregate functions) can later be disabled for adjustment.
  std::vector<TableExprNode> applySelNodes_p;
  // # The resulting table.
  Table table_p;
  // # The table resulting from a projection with expressions.
  Table projectExprTable_p;
  // # The resulting row numbers.
  Vector<rownr_t> rownrs_p;
};
 
}  // namespace casacore
 
#endif