1#@!#!123s

: / opt / saltstack / salt / lib / python3.10 / site-packages / salt / modules /
Filename : solrcloud.py
back
"""
Module for solrcloud configuration

.. versionadded:: 2017.7.0

For now, module is limited to http-exposed API. It doesn't implement config upload via Solr zkCli
"""

import logging

import salt.utils.http as http
from salt.exceptions import SaltInvocationError

log = logging.getLogger(__name__)

"""
Core properties type definition.
Reference: https://cwiki.apache.org/confluence/display/solr/Defining+core.properties
"""
STRING_PROPS_LIST = [
    "config",
    "schema",
    "dataDir",
    "configSet",
    "properties",
    "coreNodeName",
    "ulogDir",
    "shard",
    "collection",
    "roles",
]
BOOL_PROPS_LIST = ["transient", "loadOnStartup"]

"""
Collections options type definition
Reference: https://cwiki.apache.org/confluence/display/solr/Collections+API#CollectionsAPI-api1
"""
STRING_OPTIONS_LIST = [
    "collection.configName",
    "router.field",
    "async",
    "rule",
    "snitch",
]
INT_OPTIONS_LIST = ["numShards", "replicationFactor", "maxShardsPerNode"]
BOOL_OPTIONS_LIST = ["autoAddReplicas", "createNodeSet.shuffle"]
LIST_OPTIONS_LIST = ["shards", "createNodeSet"]
DICT_OPTIONS_LIST = ["properties"]

"""
Collection unmodifiable options
Reference: https://cwiki.apache.org/confluence/display/solr/Collections+API#CollectionsAPI-modifycoll
"""
CREATION_ONLY_OPTION = [
    "maxShardsPerNode",
    "replicationFactor",
    "autoAddReplicas",
    "collection.configName",
    "rule",
    "snitch",
]


def __virtual__():
    return "solrcloud"


def _query(url, solr_url="http://localhost:8983/solr/", **kwargs):
    """

    Internal function to query solrcloud

    :param url: relative solr URL
    :param solr_url: solr base URL
    :param kwargs: additional args passed to http.query call
    :return: Query JSON answer converted to python dict
    :rtype: dict

    """

    if not isinstance(solr_url, str):
        raise ValueError("solr_url must be a string")

    if solr_url[-1:] != "/":
        solr_url = solr_url + "/"

    query_result = http.query(
        solr_url + url, decode_type="json", decode=True, raise_error=True, **kwargs
    )
    if "error" in query_result:
        if query_result["status"] == 404:
            raise SaltInvocationError(
                "Got a 404 when trying to contact solr at {solr_url}{url}. Please check"
                " your solr URL.".format(solr_url=solr_url, url=url)
            )
        else:
            raise SaltInvocationError(
                "Got a {status} error when calling {solr_url}{url} : {error}".format(
                    status=str(query_result["status"]),
                    solr_url=solr_url,
                    url=url,
                    error=query_result["error"],
                )
            )
    else:
        return query_result["dict"]


def _validate_core_properties(properties):
    """

    Internal function to validate core properties

    """

    props_string = ""

    for prop_name, prop_value in properties.items():
        if prop_name in BOOL_PROPS_LIST:
            if not isinstance(prop_value, bool):
                raise ValueError('Option "' + prop_name + '" value must be an boolean')

            props_string = (
                props_string
                + "&property."
                + prop_name
                + "="
                + ("true" if prop_value else "false")
            )
        elif prop_name in STRING_PROPS_LIST:
            if not isinstance(prop_value, str):
                raise ValueError(
                    'In option "properties", core property "'
                    + prop_name
                    + '" value must be a string'
                )

            props_string = props_string + "&property." + prop_name + "=" + prop_value

        else:
            props_string = (
                props_string + "&property." + str(prop_name) + "=" + str(prop_value)
            )

    return props_string


def _validate_collection_options(options):
    """

    Internal function to validate collections options

    """

    options_string = ""

    for option_name, option_value in options.items():
        if option_name in STRING_OPTIONS_LIST:
            if not isinstance(option_value, str):
                raise ValueError('Option "' + option_name + '" value must be a string')

            options_string = options_string + "&" + option_name + "=" + option_value

        elif option_name in INT_OPTIONS_LIST:
            if not isinstance(option_value, int):
                raise ValueError('Option "' + option_name + '" value must be an int')

            options_string = (
                options_string + "&" + option_name + "=" + str(option_value)
            )

        elif option_name in BOOL_OPTIONS_LIST:
            if not isinstance(option_value, bool):
                raise ValueError(
                    'Option "' + option_name + '" value must be an boolean'
                )

            options_string = (
                options_string
                + "&"
                + option_name
                + "="
                + ("true" if option_value else "false")
            )

        elif option_name in LIST_OPTIONS_LIST:
            if not isinstance(option_value, list):
                raise ValueError(
                    'Option "' + option_name + '" value must be a list of strings'
                )

            options_string = (
                options_string + "&" + option_name + "=" + ", ".join(option_value)
            )

        elif option_name in DICT_OPTIONS_LIST:
            if not isinstance(option_value, dict):
                raise ValueError('Option "' + option_name + '" value must be an dict')

            options_string = options_string + _validate_core_properties(option_value)
        else:
            raise ValueError('Unknown option "' + option_name + '"')

    return options_string


def collection_creation_options():
    """

    Get collection option list that can only be defined at creation

    CLI Example:

    .. code-block:: bash

        salt '*' solrcloud.collection_creation_options
    """

    return CREATION_ONLY_OPTION


def cluster_status(**kwargs):
    """

    Get cluster status

    Additional parameters (kwargs) may be passed, they will be proxied to http.query

    CLI Example:

    .. code-block:: bash

        salt '*' solrcloud.cluster_status
    """

    return _query("admin/collections?action=CLUSTERSTATUS&wt=json", **kwargs)["cluster"]


def alias_exists(alias_name, **kwargs):
    """

    Check alias existence

    Additional parameters (kwargs) may be passed, they will be proxied to http.query

    CLI Example:

    .. code-block:: bash

        salt '*' solrcloud.alias_exists my_alias
    """

    if not isinstance(alias_name, str):
        raise ValueError("Alias name must be a string")

    cluster = cluster_status(**kwargs)

    return "aliases" in cluster and alias_name in cluster["aliases"]


def alias_get_collections(alias_name, **kwargs):
    """

    Get collection list for an alias

    Additional parameters (kwargs) may be passed, they will be proxied to http.query

    CLI Example:

    .. code-block:: bash

        salt '*' solrcloud.alias_get my_alias
    """

    if not isinstance(alias_name, str):
        raise ValueError("Alias name must be a string")

    collection_aliases = [
        (k_v[0], k_v[1]["aliases"])
        for k_v in cluster_status(**kwargs)["collections"].items()
        if "aliases" in k_v[1]
    ]
    aliases = [
        k_v1[0] for k_v1 in [k_v for k_v in collection_aliases if alias_name in k_v[1]]
    ]

    return aliases


def alias_set_collections(alias_name, collections=None, **kwargs):
    """

    Define an alias

    Additional parameters (kwargs) may be passed, they will be proxied to http.query

    CLI Example:

    .. code-block:: bash

        salt '*' solrcloud.alias_set my_alias collections=[collection1, colletion2]
    """

    if not isinstance(collections, list):
        raise SaltInvocationError(
            "Collection parameter must be defined and contain a list of collection name"
        )

    for collection in collections:
        if not isinstance(collection, str):
            raise ValueError("Collection name must be a string")

    return _query(
        "admin/collections?action=CREATEALIAS&name={alias}&wt=json&collections={collections}".format(
            alias=alias_name, collections=", ".join(collections)
        ),
        **kwargs
    )


def collection_reload(collection, **kwargs):
    """

    Check if a collection exists

    Additional parameters (kwargs) may be passed, they will be proxied to http.query

    CLI Example:

    .. code-block:: bash

        salt '*' solrcloud.collection_reload collection_name

    """

    _query(
        "admin/collections?action=RELOAD&name={collection}&wt=json".format(
            collection=collection
        ),
        **kwargs
    )


def collection_list(**kwargs):
    """

    List all collections

    Additional parameters (kwargs) may be passed, they will be proxied to http.query

    CLI Example:

    .. code-block:: bash

        salt '*' solrcloud.collection_list

    """

    return _query("admin/collections?action=LIST&wt=json", **kwargs)["collections"]


def collection_exists(collection_name, **kwargs):
    """

    Check if a collection exists

    Additional parameters (kwargs) may be passed, they will be proxied to http.query

    CLI Example:

    .. code-block:: bash

        salt '*' solrcloud.collection_exists collection_name

    """

    if not isinstance(collection_name, str):
        raise ValueError("Collection name must be a string")

    return collection_name in collection_list(**kwargs)


def collection_backup(collection_name, location, backup_name=None, **kwargs):
    """

    Create a backup for a collection.

    Additional parameters (kwargs) may be passed, they will be proxied to http.query

    CLI Example:

    .. code-block:: bash

        salt '*' solrcloud.core_backup collection_name /mnt/nfs_backup
    """

    if not collection_exists(collection_name, **kwargs):
        raise ValueError("Collection doesn't exists")

    if backup_name is not None:
        backup_name = "&name={}".format(backup_name)
    else:
        backup_name = ""

    _query(
        "{collection}/replication?command=BACKUP&location={location}{backup_name}&wt=json".format(
            collection=collection_name, backup_name=backup_name, location=location
        ),
        **kwargs
    )


def collection_backup_all(location, backup_name=None, **kwargs):
    """

    Create a backup for all collection present on the server.

    Additional parameters (kwargs) may be passed, they will be proxied to http.query

    CLI Example:

    .. code-block:: bash

        salt '*' solrcloud.core_backup /mnt/nfs_backup
    """

    for collection_name in collection_list(**kwargs):
        if backup_name is not None:
            backup_name = "&name={backup}.{collection}".format(
                backup=backup_name, collection=collection_name
            )
        else:
            backup_name = ""

        _query(
            "{collection}/replication?command=BACKUP&location={location}{backup_name}&wt=json".format(
                collection=collection_name, backup_name=backup_name, location=location
            ),
            **kwargs
        )


def collection_create(collection_name, options=None, **kwargs):
    """

    Create a collection,

    Additional parameters (kwargs) may be passed, they will be proxied to http.query

    CLI Example:

    .. code-block:: bash

        salt '*' solrcloud.collection_create collection_name

    Collection creation options may be passed using the "options" parameter.
    Do not include option "name" since it already specified by the mandatory parameter "collection_name"

    .. code-block:: bash

        salt '*' solrcloud.collection_create collection_name options={"replicationFactor":2, "numShards":3}

    Cores options may be passed using the "properties" key in options.
    Do not include property "name"

    .. code-block:: bash

        salt '*' solrcloud.collection_create collection_name options={"replicationFactor":2, "numShards":3, \
            "properties":{"dataDir":"/srv/solr/hugePartitionSollection"}}
    """

    if options is None:
        options = {}

    if not isinstance(options, dict):
        raise SaltInvocationError("options parameter must be a dictionary")

    options_string = _validate_collection_options(options)

    _query(
        "admin/collections?action=CREATE&wt=json&name="
        + collection_name
        + options_string,
        **kwargs
    )


def collection_check_options(options):
    """
    Check collections options

    CLI Example:

    .. code-block:: bash

        salt '*' solrcloud.collection_check_options '{"replicationFactor":4}'
    """

    try:
        _validate_collection_options(options)
        return True
    except ValueError:
        return False


def collection_get_options(collection_name, **kwargs):
    """
    Get collection options

    Additional parameters (kwargs) may be passed, they will be proxied to http.query

    CLI Example:

    .. code-block:: bash

        salt '*' solrcloud.collection_get_options collection_name
    """

    cluster = cluster_status(**kwargs)

    options = {
        "collection.configName": cluster["collections"][collection_name]["configName"],
        "router.name": cluster["collections"][collection_name]["router"]["name"],
        "replicationFactor": int(
            cluster["collections"][collection_name]["replicationFactor"]
        ),
        "maxShardsPerNode": int(
            cluster["collections"][collection_name]["maxShardsPerNode"]
        ),
        "autoAddReplicas": cluster["collections"][collection_name]["autoAddReplicas"]
        is True,
    }

    if "rule" in cluster["collections"][collection_name]:
        options["rule"] = cluster["collections"][collection_name]["rule"]
    if "snitch" in cluster["collections"][collection_name]:
        options["snitch"] = cluster["collections"][collection_name]["rule"]

    return options


def collection_set_options(collection_name, options, **kwargs):
    """
    Change collection options

    Additional parameters (kwargs) may be passed, they will be proxied to http.query

    Note that not every parameter can be changed after collection creation

    CLI Example:

    .. code-block:: bash

        salt '*' solrcloud.collection_set_options collection_name options={"replicationFactor":4}
    """

    for option in list(options.keys()):
        if option not in CREATION_ONLY_OPTION:
            raise ValueError(
                "Option " + option + " can't be modified after collection creation."
            )

    options_string = _validate_collection_options(options)

    _query(
        "admin/collections?action=MODIFYCOLLECTION&wt=json&collection="
        + collection_name
        + options_string,
        **kwargs
    )