Integrations API¶

AnnData and Scanpy integration modules.

AnnData Integration¶

`slaf.integrations.anndata` ¶

Classes¶

`LazyExpressionMatrix` ¶

Bases: LazySparseMixin

Lazy expression matrix backed by SLAF with scipy.sparse interface.

LazyExpressionMatrix provides a scipy.sparse-compatible interface for accessing single-cell expression data stored in SLAF format. It implements lazy evaluation to avoid loading all data into memory, making it suitable for large datasets.

Key Features

scipy.sparse-compatible interface
Lazy evaluation for memory efficiency
Caching for repeated queries
Support for cell and gene subsetting
Integration with AnnData objects

Examples:

>>> # Basic usage
>>> slaf_array = SLAFArray("path/to/data.slaf")
>>> lazy_matrix = LazyExpressionMatrix(slaf_array)
>>> print(f"Matrix shape: {lazy_matrix.shape}")
Matrix shape: (1000, 20000)

>>> # With AnnData integration
>>> adata = LazyAnnData(slaf_array)
>>> matrix = adata.X
>>> print(f"Expression matrix shape: {matrix.shape}")
Expression matrix shape: (1000, 20000)

>>> # Subsetting operations
>>> subset_matrix = matrix[:100, :5000]  # First 100 cells, first 5000 genes
>>> print(f"Subset shape: {subset_matrix.shape}")
Subset shape: (100, 5000)

Source code in slaf/integrations/anndata.py

class LazyExpressionMatrix(LazySparseMixin):
    """
    Lazy expression matrix backed by SLAF with scipy.sparse interface.

    LazyExpressionMatrix provides a scipy.sparse-compatible interface for accessing
    single-cell expression data stored in SLAF format. It implements lazy evaluation
    to avoid loading all data into memory, making it suitable for large datasets.

    Key Features:
        - scipy.sparse-compatible interface
        - Lazy evaluation for memory efficiency
        - Caching for repeated queries
        - Support for cell and gene subsetting
        - Integration with AnnData objects

    Examples:
        >>> # Basic usage
        >>> slaf_array = SLAFArray("path/to/data.slaf")
        >>> lazy_matrix = LazyExpressionMatrix(slaf_array)
        >>> print(f"Matrix shape: {lazy_matrix.shape}")
        Matrix shape: (1000, 20000)

        >>> # With AnnData integration
        >>> adata = LazyAnnData(slaf_array)
        >>> matrix = adata.X
        >>> print(f"Expression matrix shape: {matrix.shape}")
        Expression matrix shape: (1000, 20000)

        >>> # Subsetting operations
        >>> subset_matrix = matrix[:100, :5000]  # First 100 cells, first 5000 genes
        >>> print(f"Subset shape: {subset_matrix.shape}")
        Subset shape: (100, 5000)
    """

    def __init__(
        self,
        slaf_array: SLAFArray,
        table_name: str = "expression",
        layer_name: str | None = None,
    ):
        """
        Initialize lazy expression matrix with SLAF array.

        Args:
            slaf_array: SLAFArray instance containing the single-cell data.
                       Used for database queries and metadata access.
            table_name: Table name to query ("expression" or "layers"). Default: "expression"
            layer_name: Layer name for layers table (required when table_name="layers").
                       Default: None

        Examples:
            >>> # Basic initialization
            >>> slaf_array = SLAFArray("path/to/data.slaf")
            >>> matrix = LazyExpressionMatrix(slaf_array)
            >>> print(f"Initialized with shape: {matrix.shape}")
            Initialized with shape: (1000, 20000)

            >>> # Check parent reference
            >>> print(f"Parent adata: {matrix.parent_adata}")
            Parent adata: None

            >>> # Initialize for layers table
            >>> layer_matrix = LazyExpressionMatrix(slaf_array, table_name="layers", layer_name="spliced")
            >>> print(f"Layer matrix shape: {layer_matrix.shape}")
            Layer matrix shape: (1000, 20000)
        """
        super().__init__()
        self.slaf_array = slaf_array
        self.table_name = table_name
        self.layer_name = layer_name
        self.parent_adata: LazyAnnData | None = None
        # Store slicing selectors
        self._cell_selector: Any = None
        self._gene_selector: Any = None
        # Initialize shape attribute (required by LazySparseMixin)
        self._shape = self.slaf_array.shape
        self._cache: dict[str, Any] = {}  # Simple caching for repeated queries
        # Validate parameters
        if table_name == "layers" and layer_name is None:
            raise ValueError("layer_name must be provided when table_name='layers'")

    @property
    def shape(self) -> tuple[int, int]:
        """
        Shape of the expression matrix.

        Returns:
            Tuple of (n_cells, n_genes) representing the matrix dimensions.

        Examples:
            >>> slaf_array = SLAFArray("path/to/data.slaf")
            >>> matrix = LazyExpressionMatrix(slaf_array)
            >>> print(f"Matrix shape: {matrix.shape}")
            Matrix shape: (1000, 20000)
        """
        return self._shape

    @property
    def obs_names(self) -> pd.Index | None:
        """
        Cell names (observations).

        Returns:
            pandas.Index of cell names if parent AnnData is available, None otherwise.

        Examples:
            >>> slaf_array = SLAFArray("path/to/data.slaf")
            >>> adata = LazyAnnData(slaf_array)
            >>> matrix = adata.X
            >>> print(f"Cell names: {len(matrix.obs_names)}")
            Cell names: 1000
        """
        if hasattr(self, "parent_adata") and self.parent_adata is not None:
            return self.parent_adata.obs_names
        return None

    @property
    def var_names(self) -> pd.Index | None:
        """
        Gene names (variables).

        Returns:
            pandas.Index of gene names if parent AnnData is available, None otherwise.

        Examples:
            >>> slaf_array = SLAFArray("path/to/data.slaf")
            >>> adata = LazyAnnData(slaf_array)
            >>> matrix = adata.X
            >>> print(f"Gene names: {len(matrix.var_names)}")
            Gene names: 20000
        """
        if hasattr(self, "parent_adata") and self.parent_adata is not None:
            return self.parent_adata.var_names
        return None

    def _update_shape(self):
        """Update the shape attribute based on current selectors"""
        if self._cell_selector is not None or self._gene_selector is not None:
            # Calculate the shape based on selectors
            cell_selector = (
                self._cell_selector if self._cell_selector is not None else slice(None)
            )
            gene_selector = (
                self._gene_selector if self._gene_selector is not None else slice(None)
            )

            # Use the same logic as _get_result_shape in LazySparseMixin
            cell_selector = self._compose_selectors(cell_selector, None, axis=0)
            gene_selector = self._compose_selectors(gene_selector, None, axis=1)

            n_cells = self._calculate_selected_count(cell_selector, axis=0)
            n_genes = self._calculate_selected_count(gene_selector, axis=1)

            self._shape = (n_cells, n_genes)
        else:
            # No selectors applied, return original shape
            self._shape = self.slaf_array.shape

    def _calculate_selected_count(self, selector, axis: int) -> int:
        """Calculate the number of selected entities for a given selector"""
        if selector is None or (
            isinstance(selector, slice) and selector == slice(None)
        ):
            return self.slaf_array.shape[axis]

        if isinstance(selector, slice):
            start = selector.start or 0
            stop = selector.stop or self.slaf_array.shape[axis]
            step = selector.step or 1

            # Clamp bounds to actual data size
            start = max(0, min(start, self.slaf_array.shape[axis]))
            stop = max(0, min(stop, self.slaf_array.shape[axis]))

            return len(range(start, stop, step))
        elif isinstance(selector, list | np.ndarray):
            if isinstance(selector, np.ndarray) and selector.dtype == bool:
                return np.sum(selector)
            return len(selector)
        elif isinstance(selector, int | np.integer):
            return 1
        else:
            return self.slaf_array.shape[axis]

    def __getitem__(self, key) -> "LazyExpressionMatrix":
        """
        Lazy slicing - returns a new LazyExpressionMatrix with composed selectors
        No computation happens until .compute() is called
        """
        cell_selector, gene_selector = self._parse_key(key)

        # Handle selectors directly as integer indices
        # For slices, lists, ints - these are already relative to the current view
        # We need to compose them with existing selectors

        # Create a new LazyExpressionMatrix with composed selectors
        new_matrix = LazyExpressionMatrix(
            self.slaf_array, table_name=self.table_name, layer_name=self.layer_name
        )
        new_matrix.parent_adata = self.parent_adata

        # Compose selectors (these are relative to current view)
        new_matrix._cell_selector = self._compose_selectors(
            self._cell_selector, cell_selector, axis=0
        )
        new_matrix._gene_selector = self._compose_selectors(
            self._gene_selector, gene_selector, axis=1
        )

        # Update shape based on new selectors
        new_matrix._update_shape()

        return new_matrix

    def _compose_selectors(self, old, new, axis):
        """Compose two selectors (helper method)"""
        if old is None:
            return new
        if new is None or (isinstance(new, slice) and new == slice(None)):
            return old

        # If old is a slice, we need to apply new to the range defined by old
        if isinstance(old, slice):
            # Get the range from the old slice
            old_start = old.start or 0
            old_stop = old.stop or self.slaf_array.shape[axis]
            old_step = old.step or 1

            # Handle negative indices in old slice
            if old_start < 0:
                old_start = self.slaf_array.shape[axis] + old_start
            if old_stop < 0:
                old_stop = self.slaf_array.shape[axis] + old_stop

            # Clamp old slice bounds
            old_start = max(0, min(old_start, self.slaf_array.shape[axis]))
            old_stop = max(0, min(old_stop, self.slaf_array.shape[axis]))

            # Create the range from the old slice
            old_range = list(range(old_start, old_stop, old_step))

            # Now apply the new selector to this range
            if isinstance(new, slice):
                # Apply new slice to the old range
                new_start = new.start or 0
                new_stop = new.stop or len(old_range)
                new_step = new.step or 1

                # Handle negative indices in new slice
                if new_start < 0:
                    new_start = len(old_range) + new_start
                if new_stop < 0:
                    new_stop = len(old_range) + new_stop

                # Clamp new slice bounds
                new_start = max(0, min(new_start, len(old_range)))
                new_stop = max(0, min(new_stop, len(old_range)))

                # Apply the new slice to the old range
                result_indices = old_range[new_start:new_stop:new_step]
                return result_indices
            elif isinstance(new, int | np.integer):
                # Single index into the old range
                if 0 <= new < len(old_range):
                    return [old_range[new]]
                else:
                    return []
            elif isinstance(new, list | np.ndarray):
                # List of indices into the old range
                result = []
                for idx in new:
                    if 0 <= idx < len(old_range):
                        result.append(old_range[idx])
                return result
            else:
                return new

        # If old is a list of indices, apply new to those indices
        elif isinstance(old, list | np.ndarray):
            if isinstance(new, slice):
                # Apply slice to the old list
                new_start = new.start or 0
                new_stop = new.stop or len(old)
                new_step = new.step or 1

                # Handle negative indices
                if new_start < 0:
                    new_start = len(old) + new_start
                if new_stop < 0:
                    new_stop = len(old) + new_stop

                # Clamp bounds
                new_start = max(0, min(new_start, len(old)))
                new_stop = max(0, min(new_stop, len(old)))

                return old[new_start:new_stop:new_step]
            elif isinstance(new, int | np.integer):
                # Single index into the old list
                if 0 <= new < len(old):
                    return [old[new]]
                else:
                    return []
            elif isinstance(new, list | np.ndarray):
                # List of indices into the old list
                result = []
                for idx in new:
                    if 0 <= idx < len(old):
                        result.append(old[idx])
                return result
            else:
                return new

        # For other cases, return the new selector as fallback
        return new

    def _estimate_slice_size(self, cell_selector, gene_selector) -> int:
        """Estimate the size of the slice for strategy selection"""
        cell_count = self._estimate_selected_count(cell_selector, axis=0)
        gene_count = self._estimate_selected_count(gene_selector, axis=1)
        return cell_count * gene_count

    def _apply_transformations(
        self,
        matrix: scipy.sparse.csr_matrix,
        cell_selector,
        gene_selector,
    ) -> scipy.sparse.csr_matrix:
        """Apply any stored transformations to the matrix"""
        # Get transformations from the parent LazyAnnData if available
        if self.parent_adata is not None and hasattr(
            self.parent_adata, "_transformations"
        ):
            transformations = self.parent_adata._transformations
        else:
            return matrix

        # Avoid copying if no transformations
        if not transformations:
            return matrix

        # Try to apply transformations at SQL level first
        sql_transformed = self._apply_sql_transformations(
            cell_selector, gene_selector, transformations
        )

        if sql_transformed is not None:
            # SQL transformations were applied, reconstruct matrix
            return self._reconstruct_sparse_matrix(
                sql_transformed, cell_selector, gene_selector
            )

        # Fall back to numpy transformations
        return self._apply_numpy_transformations(
            matrix, cell_selector, gene_selector, transformations
        )

    def _apply_sql_transformations(
        self, cell_selector, gene_selector, transformations
    ) -> pd.DataFrame | None:
        """Apply transformations at SQL level when possible"""
        # Check if we can apply all transformations in SQL
        sql_applicable = []
        numpy_needed = []

        for transform_name, transform_data in transformations.items():
            if transform_name == "normalize_total":
                # normalize_total can be applied in SQL
                sql_applicable.append((transform_name, transform_data))
            elif transform_name == "log1p":
                # log1p can be applied in SQL
                sql_applicable.append((transform_name, transform_data))
            else:
                # Unknown transformation, needs numpy
                numpy_needed.append((transform_name, transform_data))

        # If we have numpy-only transformations, don't use SQL
        if numpy_needed:
            return None

        # Build SQL query with transformations
        # Convert list of tuples to dictionary for _build_transformed_query
        sql_transformations = dict(sql_applicable)
        query = self._build_transformed_query(
            cell_selector, gene_selector, sql_transformations
        )

        try:
            # Execute the transformed query
            return self.slaf_array.query(query)
        except Exception:
            # If SQL transformation fails, fall back to numpy
            return None

    def _build_transformed_query(
        self, cell_selector, gene_selector, transformations
    ) -> str:
        """Build SQL query with transformations applied"""
        # Build base SQL query string
        base_query = self._build_submatrix_sql(cell_selector, gene_selector)

        # Apply transformations in order
        transformed_query = base_query
        for transform_name, transform_data in transformations.items():
            if transform_name == "normalize_total":
                transformed_query = self._apply_sql_normalize_total(
                    transformed_query, transform_data
                )
            elif transform_name == "log1p":
                transformed_query = self._apply_sql_log1p(transformed_query)

        return transformed_query

    def _build_submatrix_sql(self, cell_selector, gene_selector) -> str:
        """Build SQL query string for submatrix selection"""
        # Use the QueryOptimizer to build the SQL string
        from slaf.core.query_optimizer import QueryOptimizer

        return QueryOptimizer.build_submatrix_query(
            cell_selector=cell_selector,
            gene_selector=gene_selector,
            cell_count=self.slaf_array.shape[0],  # Use original dataset dimensions
            gene_count=self.slaf_array.shape[1],  # Use original dataset dimensions
            table_name=self.table_name,
            layer_name=self.layer_name,
        )

    def _apply_sql_normalize_total(self, query: str, transform_data: dict) -> str:
        """Apply normalize_total transformation in SQL"""
        cell_factors = transform_data["cell_factors"]
        # target_sum = transform_data["target_sum"]  # Removed unused variable

        # Create a CASE statement for cell factors
        case_statements = []
        for cell_id, factor in cell_factors.items():
            # Convert scientific notation to regular decimal format for SQL compatibility
            factor_str = (
                f"{factor:.10f}".rstrip("0").rstrip(".") if factor != 0 else "0"
            )

            # Handle both integer and string cell IDs
            if isinstance(cell_id, str) and cell_id.startswith("cell_"):
                # Extract integer from string like "cell_0" -> 0
                try:
                    cell_integer_id_int = int(cell_id.split("_")[1])
                except (ValueError, IndexError):
                    # Fallback: skip this cell if we can't parse it
                    continue
            elif isinstance(cell_id, int | np.integer):
                cell_integer_id_int = int(cell_id)
            else:
                # Skip if we can't handle this cell ID type
                continue

            case_statements.append(
                f"WHEN e.cell_integer_id = {cell_integer_id_int} THEN {factor_str}"
            )

        factor_case = "CASE " + " ".join(case_statements) + " ELSE 1.0 END"

        # Wrap the query to apply normalization
        return f"""
        SELECT
            e.cell_integer_id,
            e.gene_integer_id,
            e.value * {factor_case} as value
        FROM ({query}) as base_data e
        """

    def _apply_sql_log1p(self, query: str) -> str:
        """Apply log1p transformation in SQL, only to nonzero values (sparse semantics)"""
        return f"""
        SELECT
            e.cell_integer_id,
            e.gene_integer_id,
            CASE WHEN e.value != 0 THEN LN(1 + e.value) ELSE 0 END as value
        FROM ({query}) as base_data e
        """

    def _apply_numpy_transformations(
        self,
        matrix: scipy.sparse.csr_matrix,
        cell_selector,
        gene_selector,
        transformations,
    ) -> scipy.sparse.csr_matrix:
        """Apply transformations using numpy operations"""
        # Apply transformations in the order they were added (preserves order)
        result = matrix
        for transform_name, transform_data in transformations.items():
            if transform_name == "normalize_total":
                result = self._apply_normalize_total(
                    result, cell_selector, transform_data
                )
            elif transform_name == "log1p":
                result = self._apply_log1p(result)

        return result

    def _apply_normalize_total(
        self,
        matrix: scipy.sparse.csr_matrix,
        cell_selector,
        transform_data,
    ) -> scipy.sparse.csr_matrix:
        """Apply normalize_total transformation using vectorized operations"""
        cell_factors = transform_data.get("cell_factors", {})
        obs_names_local = []  # Always a list
        obs_names = None
        if hasattr(self, "parent_adata") and self.parent_adata is not None:
            try:
                obs_names = self.parent_adata.obs_names
            except (AttributeError, TypeError):
                obs_names = None
        if obs_names is None or not isinstance(obs_names, list | np.ndarray | pd.Index):
            if (
                matrix is not None
                and hasattr(matrix, "shape")
                and matrix.shape is not None
            ):
                obs_names_local = [f"cell_{i}" for i in range(matrix.shape[0])]
            else:
                obs_names_local = []
        else:
            obs_names_local = list(obs_names)
        # Now obs_names_local is always a list
        # Determine selected_cell_names based on cell_selector
        if cell_selector is None or (
            isinstance(cell_selector, slice) and cell_selector == slice(None)
        ):
            selected_cell_names = obs_names_local
        elif isinstance(cell_selector, slice):
            start = cell_selector.start or 0
            stop = cell_selector.stop or len(obs_names_local)
            step = cell_selector.step or 1
            # Clamp bounds
            start = max(0, min(start, len(obs_names_local)))
            stop = max(0, min(stop, len(obs_names_local)))
            selected_cell_names = obs_names_local[start:stop:step]
        elif isinstance(cell_selector, list | np.ndarray):
            if len(obs_names_local) > 0:
                if (
                    isinstance(cell_selector, np.ndarray)
                    and cell_selector.dtype == bool
                ):
                    selected_cell_names = [
                        obs_names_local[i]
                        for i, keep in enumerate(cell_selector)
                        if keep and 0 <= i < len(obs_names_local)
                    ]
                else:
                    selected_cell_names = [
                        obs_names_local[i]
                        for i in cell_selector
                        if isinstance(i, int | np.integer)
                        and 0 <= i < len(obs_names_local)
                    ]
            else:
                selected_cell_names = []
        elif isinstance(cell_selector, int | np.integer):
            if 0 <= cell_selector < len(obs_names_local):
                selected_cell_names = [obs_names_local[cell_selector]]
            else:
                selected_cell_names = []
        else:
            selected_cell_names = obs_names_local
        # Create a vector of factors for all cells at once
        cell_factors_vector = np.array(
            [cell_factors.get(name, 1.0) for name in selected_cell_names]
        )
        # Apply vectorized scaling using CSR matrix properties
        # Create a copy only if we need to modify the data
        if not np.allclose(cell_factors_vector, 1.0):
            result = matrix.copy()
            # Apply scaling to each row using vectorized operations
            for i in range(result.shape[0]):
                if i < len(cell_factors_vector):
                    factor = cell_factors_vector[i]
                    if factor != 1.0:
                        # Scale the non-zero elements in this row
                        start_idx = result.indptr[i]
                        end_idx = result.indptr[i + 1]
                        result.data[start_idx:end_idx] *= factor
            return result
        else:
            # No scaling needed, return original matrix
            return matrix

    def _apply_log1p(self, matrix: scipy.sparse.csr_matrix) -> scipy.sparse.csr_matrix:
        """Apply log1p transformation using vectorized operations"""
        # Create a copy only if we need to modify the data
        result = matrix.copy()
        # Apply log1p to all non-zero values using vectorized operation
        result.data = np.log1p(result.data)
        return result

    def __array_function__(self, func, types, args, kwargs):
        """Intercept numpy functions for lazy evaluation"""
        if func == np.mean:
            axis = kwargs.get("axis", None)
            return self.mean(axis=axis)
        elif func == np.sum:
            axis = kwargs.get("axis", None)
            return self.sum(axis=axis)
        elif func == np.var:
            axis = kwargs.get("axis", None)
            return self.var(axis=axis)
        elif func == np.std:
            axis = kwargs.get("axis", None)
            return self.std(axis=axis)
        else:
            # Fall back to materializing the matrix
            matrix = self.compute()
            return func(matrix, *args[1:], **kwargs)

    def mean(
        self, axis: int | None = None, fragments: bool | None = None
    ) -> float | np.ndarray:
        """Compute mean along axis via SQL aggregation"""
        return self._aggregation_with_fragments("mean", fragments, axis=axis)

    def sum(
        self, axis: int | None = None, fragments: bool | None = None
    ) -> float | np.ndarray:
        """Compute sum along axis via SQL aggregation"""
        return self._aggregation_with_fragments("sum", fragments, axis=axis)

    def var(self, axis: int | None = None) -> float | np.ndarray:
        """Compute variance along axis via SQL aggregation"""
        return self._sql_aggregation("variance", axis)

    def std(self, axis: int | None = None) -> float | np.ndarray:
        """Compute standard deviation along axis"""
        return self._sql_aggregation("stddev", axis)

    def toarray(self) -> np.ndarray:
        """Convert to dense numpy array"""
        matrix = self.compute()
        return matrix.toarray()

    def compute(self, fragments: bool | None = None) -> scipy.sparse.csr_matrix:
        """
        Explicitly compute the matrix with all transformations applied.
        This is where the actual SQL query and materialization happens.

        Args:
            fragments: Whether to use fragment processing (None for automatic)

        Returns:
            Sparse matrix with transformations applied
        """
        # Determine processing strategy
        if fragments is not None:
            use_fragments = fragments
        else:
            # Check if dataset has multiple fragments
            try:
                fragments_list = self.slaf_array.expression.get_fragments()
                use_fragments = len(fragments_list) > 1
            except Exception:
                use_fragments = False

        # Fragment processing now supports both expression and layers tables
        if use_fragments:
            processor = FragmentProcessor(
                self.slaf_array,
                cell_selector=self._cell_selector,
                gene_selector=self._gene_selector,
                max_workers=4,
                enable_caching=True,
                table_name=self.table_name,
                layer_name=self.layer_name,
            )
            # Use smart strategy selection for optimal performance
            lazy_pipeline = processor.build_lazy_pipeline_smart("compute_matrix")
            result = processor.compute(lazy_pipeline)
            return self._convert_to_sparse_matrix(result)
        else:
            return self._compute_global()

    def _compute_global(self) -> scipy.sparse.csr_matrix:
        """Compute the matrix globally (original implementation)"""
        # Build the SQL query with transformations
        if self.parent_adata is not None and hasattr(
            self.parent_adata, "_transformations"
        ):
            transformations = self.parent_adata._transformations
        else:
            transformations = {}

        # For layers table, use Lance's native filter pushdown instead of SQL to avoid Substrait issues
        if self.table_name == "layers":
            base_result = self._query_layers_with_lance()
        else:
            # Try SQL-level transformations first
            if transformations:
                sql_result = self._apply_sql_transformations(
                    self._cell_selector, self._gene_selector, transformations
                )
                if sql_result is not None:
                    return self._reconstruct_sparse_matrix(
                        sql_result, self._cell_selector, self._gene_selector
                    )

            # Fall back to base query + numpy transformations
            base_query = self._build_submatrix_sql(
                self._cell_selector, self._gene_selector
            )
            base_result = self.slaf_array.query(base_query)

        # Reconstruct base matrix
        base_matrix = self._reconstruct_sparse_matrix(
            base_result, self._cell_selector, self._gene_selector
        )

        # Apply transformations in numpy if needed
        if transformations:
            return self._apply_numpy_transformations(
                base_matrix,
                self._cell_selector,
                self._gene_selector,
                transformations,
            )

        return base_matrix

    def _query_layers_with_lance(self) -> pl.DataFrame:
        """
        Query layers table using Lance's native filter pushdown (avoids Substrait issues).

        This method uses Lance's native `to_table()` with SQL filter strings, which
        provides efficient filter pushdown without the Substrait casting issues that
        occur with Polars' scan_pyarrow_dataset.

        According to https://lance.org/guide/read_and_write/#reading-lance-dataset,
        Lance supports native SQL filter pushdown which is more efficient and reliable
        than going through Polars' PyArrow dataset scanning.

        Args:
            Returns Polars DataFrame with columns: cell_integer_id, gene_integer_id, value
        """
        if self.slaf_array.layers is None:
            raise ValueError("Layers table not available in this dataset")
        if self.layer_name is None:
            raise ValueError("layer_name must be provided when table_name='layers'")

        # Build SQL filter conditions for cell and gene selectors
        cell_condition = self._selector_to_sql_condition(
            self._cell_selector, axis=0, entity_type="cell"
        )
        gene_condition = self._selector_to_sql_condition(
            self._gene_selector, axis=1, entity_type="gene"
        )

        # Compose filter string (combine cell and gene conditions with AND)
        filter_parts = []
        if cell_condition != "TRUE":
            filter_parts.append(cell_condition)
        if gene_condition != "TRUE":
            filter_parts.append(gene_condition)
        # Also filter out NULL values for the layer column (sparse representation)
        filter_parts.append(f"{self.layer_name} IS NOT NULL")

        # Compose final filter string
        if filter_parts:
            filter_str = " AND ".join(filter_parts)
        else:
            filter_str = f"{self.layer_name} IS NOT NULL"

        # Use Lance's native to_table() with filter pushdown
        # Select only the columns we need
        table = self.slaf_array.layers.to_table(
            columns=["cell_integer_id", "gene_integer_id", self.layer_name],
            filter=filter_str,
        )

        # Convert to Polars DataFrame
        # Type cast: to_table() with multiple columns always returns a DataFrame
        df: pl.DataFrame = pl.from_arrow(table)  # type: ignore[assignment]

        # Rename layer column to "value" for consistency with expression table
        df = df.rename({self.layer_name: "value"})

        return df

    def _convert_to_sparse_matrix(
        self, result_df: pl.DataFrame
    ) -> scipy.sparse.csr_matrix:
        """
        Convert fragment processing result to sparse matrix.

        Args:
            result_df: Polars DataFrame from fragment processing

        Returns:
            Sparse matrix representation
        """
        if len(result_df) == 0:
            # Return empty matrix with appropriate shape
            return scipy.sparse.csr_matrix(self.shape)

        # Convert to COO format for efficient sparse matrix construction
        rows = result_df["cell_integer_id"].to_numpy()
        cols = result_df["gene_integer_id"].to_numpy()
        data = result_df["value"].to_numpy()

        # Remap integer IDs to local coordinates if selectors are applied
        if self._cell_selector is not None:
            if isinstance(self._cell_selector, slice):
                start = self._cell_selector.start or 0
                if start > 0:
                    rows = rows - start
            elif isinstance(self._cell_selector, list | np.ndarray):
                # For list/array selectors, we need to create a mapping
                # But this is complex, so for now we'll assume the query already filtered correctly
                # and we just need to remap if it's a slice
                pass

        if self._gene_selector is not None:
            if isinstance(self._gene_selector, slice):
                start = self._gene_selector.start or 0
                if start > 0:
                    cols = cols - start
            elif isinstance(self._gene_selector, list | np.ndarray):
                # For list/array selectors, we need to create a mapping
                # But this is complex, so for now we'll assume the query already filtered correctly
                # and we just need to remap if it's a slice
                pass

        # Create sparse matrix
        return scipy.sparse.coo_matrix((data, (rows, cols)), shape=self.shape).tocsr()

Attributes¶

`shape: tuple[int, int]` `property` ¶

Shape of the expression matrix.

Returns:

Type	Description
`tuple[int, int]`	Tuple of (n_cells, n_genes) representing the matrix dimensions.

Examples:

>>> slaf_array = SLAFArray("path/to/data.slaf")
>>> matrix = LazyExpressionMatrix(slaf_array)
>>> print(f"Matrix shape: {matrix.shape}")
Matrix shape: (1000, 20000)

`obs_names: pd.Index | None` `property` ¶

Cell names (observations).

Returns:

Type	Description
`Index \| None`	pandas.Index of cell names if parent AnnData is available, None otherwise.

Examples:

>>> slaf_array = SLAFArray("path/to/data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> matrix = adata.X
>>> print(f"Cell names: {len(matrix.obs_names)}")
Cell names: 1000

`var_names: pd.Index | None` `property` ¶

Gene names (variables).

Returns:

Type	Description
`Index \| None`	pandas.Index of gene names if parent AnnData is available, None otherwise.

Examples:

>>> slaf_array = SLAFArray("path/to/data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> matrix = adata.X
>>> print(f"Gene names: {len(matrix.var_names)}")
Gene names: 20000

Functions¶

`init(slaf_array: SLAFArray, table_name: str = 'expression', layer_name: str | None = None)` ¶

Initialize lazy expression matrix with SLAF array.

Parameters:

Name	Type	Description	Default
`slaf_array`	`SLAFArray`	SLAFArray instance containing the single-cell data. Used for database queries and metadata access.	required
`table_name`	`str`	Table name to query ("expression" or "layers"). Default: "expression"	`'expression'`
`layer_name`	`str \| None`	Layer name for layers table (required when table_name="layers"). Default: None	`None`

Examples:

>>> # Basic initialization
>>> slaf_array = SLAFArray("path/to/data.slaf")
>>> matrix = LazyExpressionMatrix(slaf_array)
>>> print(f"Initialized with shape: {matrix.shape}")
Initialized with shape: (1000, 20000)

>>> # Check parent reference
>>> print(f"Parent adata: {matrix.parent_adata}")
Parent adata: None

>>> # Initialize for layers table
>>> layer_matrix = LazyExpressionMatrix(slaf_array, table_name="layers", layer_name="spliced")
>>> print(f"Layer matrix shape: {layer_matrix.shape}")
Layer matrix shape: (1000, 20000)

Source code in slaf/integrations/anndata.py

def __init__(
    self,
    slaf_array: SLAFArray,
    table_name: str = "expression",
    layer_name: str | None = None,
):
    """
    Initialize lazy expression matrix with SLAF array.

    Args:
        slaf_array: SLAFArray instance containing the single-cell data.
                   Used for database queries and metadata access.
        table_name: Table name to query ("expression" or "layers"). Default: "expression"
        layer_name: Layer name for layers table (required when table_name="layers").
                   Default: None

    Examples:
        >>> # Basic initialization
        >>> slaf_array = SLAFArray("path/to/data.slaf")
        >>> matrix = LazyExpressionMatrix(slaf_array)
        >>> print(f"Initialized with shape: {matrix.shape}")
        Initialized with shape: (1000, 20000)

        >>> # Check parent reference
        >>> print(f"Parent adata: {matrix.parent_adata}")
        Parent adata: None

        >>> # Initialize for layers table
        >>> layer_matrix = LazyExpressionMatrix(slaf_array, table_name="layers", layer_name="spliced")
        >>> print(f"Layer matrix shape: {layer_matrix.shape}")
        Layer matrix shape: (1000, 20000)
    """
    super().__init__()
    self.slaf_array = slaf_array
    self.table_name = table_name
    self.layer_name = layer_name
    self.parent_adata: LazyAnnData | None = None
    # Store slicing selectors
    self._cell_selector: Any = None
    self._gene_selector: Any = None
    # Initialize shape attribute (required by LazySparseMixin)
    self._shape = self.slaf_array.shape
    self._cache: dict[str, Any] = {}  # Simple caching for repeated queries
    # Validate parameters
    if table_name == "layers" and layer_name is None:
        raise ValueError("layer_name must be provided when table_name='layers'")

`mean(axis: int | None = None, fragments: bool | None = None) -> float | np.ndarray` ¶

Compute mean along axis via SQL aggregation

Source code in slaf/integrations/anndata.py

def mean(
    self, axis: int | None = None, fragments: bool | None = None
) -> float | np.ndarray:
    """Compute mean along axis via SQL aggregation"""
    return self._aggregation_with_fragments("mean", fragments, axis=axis)

`sum(axis: int | None = None, fragments: bool | None = None) -> float | np.ndarray` ¶

Compute sum along axis via SQL aggregation

Source code in slaf/integrations/anndata.py

def sum(
    self, axis: int | None = None, fragments: bool | None = None
) -> float | np.ndarray:
    """Compute sum along axis via SQL aggregation"""
    return self._aggregation_with_fragments("sum", fragments, axis=axis)

`var(axis: int | None = None) -> float | np.ndarray` ¶

Compute variance along axis via SQL aggregation

Source code in slaf/integrations/anndata.py

def var(self, axis: int | None = None) -> float | np.ndarray:
    """Compute variance along axis via SQL aggregation"""
    return self._sql_aggregation("variance", axis)

`std(axis: int | None = None) -> float | np.ndarray` ¶

Compute standard deviation along axis

Source code in slaf/integrations/anndata.py

def std(self, axis: int | None = None) -> float | np.ndarray:
    """Compute standard deviation along axis"""
    return self._sql_aggregation("stddev", axis)

`toarray() -> np.ndarray` ¶

Convert to dense numpy array

Source code in slaf/integrations/anndata.py

def toarray(self) -> np.ndarray:
    """Convert to dense numpy array"""
    matrix = self.compute()
    return matrix.toarray()

`compute(fragments: bool | None = None) -> scipy.sparse.csr_matrix` ¶

Explicitly compute the matrix with all transformations applied. This is where the actual SQL query and materialization happens.

Parameters:

Name	Type	Description	Default
`fragments`	`bool \| None`	Whether to use fragment processing (None for automatic)	`None`

Returns:

Type	Description
`csr_matrix`	Sparse matrix with transformations applied

Source code in slaf/integrations/anndata.py

def compute(self, fragments: bool | None = None) -> scipy.sparse.csr_matrix:
    """
    Explicitly compute the matrix with all transformations applied.
    This is where the actual SQL query and materialization happens.

    Args:
        fragments: Whether to use fragment processing (None for automatic)

    Returns:
        Sparse matrix with transformations applied
    """
    # Determine processing strategy
    if fragments is not None:
        use_fragments = fragments
    else:
        # Check if dataset has multiple fragments
        try:
            fragments_list = self.slaf_array.expression.get_fragments()
            use_fragments = len(fragments_list) > 1
        except Exception:
            use_fragments = False

    # Fragment processing now supports both expression and layers tables
    if use_fragments:
        processor = FragmentProcessor(
            self.slaf_array,
            cell_selector=self._cell_selector,
            gene_selector=self._gene_selector,
            max_workers=4,
            enable_caching=True,
            table_name=self.table_name,
            layer_name=self.layer_name,
        )
        # Use smart strategy selection for optimal performance
        lazy_pipeline = processor.build_lazy_pipeline_smart("compute_matrix")
        result = processor.compute(lazy_pipeline)
        return self._convert_to_sparse_matrix(result)
    else:
        return self._compute_global()

`LazyDictionaryViewMixin` ¶

Base mixin for dictionary-like views (layers, obs, var).

Provides common dictionary interface methods that are identical across all view types: layers, obs columns, and var columns.

Source code in slaf/integrations/anndata.py

class LazyDictionaryViewMixin:
    """
    Base mixin for dictionary-like views (layers, obs, var).

    Provides common dictionary interface methods that are identical across
    all view types: layers, obs columns, and var columns.
    """

    def keys(self) -> list[str]:
        """
        Return list of keys.

        This method must be implemented by subclasses.
        """
        raise NotImplementedError("Subclasses must implement keys()")

    def __contains__(self, key: str) -> bool:
        """Check if key exists"""
        return key in self.keys()

    def __len__(self) -> int:
        """Number of keys"""
        return len(self.keys())

    def __iter__(self):
        """Iterate over keys"""
        return iter(self.keys())

    def _validate_name(self, key: str):
        """
        Validate name (alphanumeric + underscore, non-empty).

        Args:
            key: Name to validate

        Raises:
            ValueError: If name is empty or contains invalid characters
        """
        if not key:
            raise ValueError("Name cannot be empty")
        if not key.replace("_", "").isalnum():
            raise ValueError(
                f"Name '{key}' contains invalid characters. "
                "Only alphanumeric characters and underscores are allowed."
            )

Functions¶

`keys() -> list[str]` ¶

Return list of keys.

This method must be implemented by subclasses.

Source code in slaf/integrations/anndata.py

def keys(self) -> list[str]:
    """
    Return list of keys.

    This method must be implemented by subclasses.
    """
    raise NotImplementedError("Subclasses must implement keys()")

`LazyLayersView` ¶

Bases: LazyDictionaryViewMixin

Dictionary-like view of layers with lazy evaluation.

LazyLayersView provides a dictionary-like interface for accessing AnnData layers stored in the layers.lance table. It supports reading layers as LazyExpressionMatrix objects and provides methods to list, check, and iterate over available layers.

Key Features

Dictionary-like interface: layers["name"], "name" in layers, len(layers)
Lazy evaluation: layers are accessed on-demand
Config.json consistency: reads from config for fast layer discovery
Backward compatibility: works with datasets without layers

Examples:

>>> # Access a layer
>>> slaf_array = SLAFArray("data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> spliced = adata.layers["spliced"]
>>> print(f"Layer shape: {spliced.shape}")
Layer shape: (1000, 20000)

>>> # List available layers
>>> print(list(adata.layers.keys()))
['spliced', 'unspliced']

>>> # Check if layer exists
>>> assert "spliced" in adata.layers
>>> assert "nonexistent" not in adata.layers

Source code in slaf/integrations/anndata.py

class LazyLayersView(LazyDictionaryViewMixin):
    """
    Dictionary-like view of layers with lazy evaluation.

    LazyLayersView provides a dictionary-like interface for accessing AnnData layers
    stored in the layers.lance table. It supports reading layers as LazyExpressionMatrix
    objects and provides methods to list, check, and iterate over available layers.

    Key Features:
        - Dictionary-like interface: layers["name"], "name" in layers, len(layers)
        - Lazy evaluation: layers are accessed on-demand
        - Config.json consistency: reads from config for fast layer discovery
        - Backward compatibility: works with datasets without layers

    Examples:
        >>> # Access a layer
        >>> slaf_array = SLAFArray("data.slaf")
        >>> adata = LazyAnnData(slaf_array)
        >>> spliced = adata.layers["spliced"]
        >>> print(f"Layer shape: {spliced.shape}")
        Layer shape: (1000, 20000)

        >>> # List available layers
        >>> print(list(adata.layers.keys()))
        ['spliced', 'unspliced']

        >>> # Check if layer exists
        >>> assert "spliced" in adata.layers
        >>> assert "nonexistent" not in adata.layers
    """

    def __init__(self, lazy_adata: "LazyAnnData"):
        """
        Initialize LazyLayersView with LazyAnnData.

        Args:
            lazy_adata: LazyAnnData instance containing the single-cell data.
        """
        self.lazy_adata = lazy_adata
        self._slaf_array = lazy_adata.slaf

    def _get_layers_from_config(self) -> list[str]:
        """Get layer names from config.json (fast path)"""
        if self._slaf_array.layers is None:
            return []

        layers_config = self._slaf_array.config.get("layers", {})
        return layers_config.get("available", [])

    def _get_layers_from_table(self) -> list[str]:
        """Get layer names by querying layers table (fallback)"""
        if self._slaf_array.layers is None:
            return []

        try:
            # Query to get distinct layer column names
            # In wide format, we need to check which columns exist
            schema = self._slaf_array.layers.schema
            column_names = [field.name for field in schema]

            # Filter out cell_integer_id and gene_integer_id
            layer_names = [
                col
                for col in column_names
                if col not in ("cell_integer_id", "gene_integer_id")
            ]
            return layer_names
        except Exception:
            return []

    def keys(self) -> list[str]:
        """List all available layer names"""
        # Fast path: read from config.json
        config_layers = self._get_layers_from_config()

        if config_layers:
            # Verify consistency with table if possible
            table_layers = self._get_layers_from_table()
            if table_layers and set(config_layers) != set(table_layers):
                # Log warning but prefer config
                import warnings

                warnings.warn(
                    f"Layer names in config.json ({config_layers}) don't match "
                    f"layers table ({table_layers}). Using config.json values.",
                    UserWarning,
                    stacklevel=2,
                )
            return config_layers

        # Fallback: query table
        return self._get_layers_from_table()

    def __getitem__(self, key: str) -> LazyExpressionMatrix:
        """Get a layer as LazyExpressionMatrix"""
        # Validate layer exists
        if key not in self.keys():
            raise KeyError(f"Layer '{key}' not found")

        # Create LazyExpressionMatrix pointing to layers table
        layer_matrix = LazyExpressionMatrix(
            self._slaf_array, table_name="layers", layer_name=key
        )
        layer_matrix.parent_adata = self.lazy_adata

        # Propagate cell/gene selectors from parent LazyAnnData
        if hasattr(self.lazy_adata, "_cell_selector"):
            layer_matrix._cell_selector = self.lazy_adata._cell_selector
        if hasattr(self.lazy_adata, "_gene_selector"):
            layer_matrix._gene_selector = self.lazy_adata._gene_selector
        layer_matrix._update_shape()

        return layer_matrix

    def _is_immutable(self, key: str) -> bool:
        """Check if a layer is immutable (converted from h5ad)"""
        layers_config = self._slaf_array.config.get("layers", {})
        immutable_layers = layers_config.get("immutable", [])
        return key in immutable_layers

    def __setitem__(
        self, key: str, value: "LazyExpressionMatrix | scipy.sparse.spmatrix"
    ):
        """
        Create or update a layer (lazy write - requires commit()).

        Stores the assignment in a pending writes queue. The actual write to
        layers.lance happens when commit() is called. This allows batching
        multiple layer operations and ensures config.json consistency.

        Args:
            key: Layer name (must be alphanumeric + underscore, non-empty)
            value: LazyExpressionMatrix or scipy sparse matrix to assign.
                   Must have the same shape as adata.X.

        Raises:
            ValueError: If layer name is invalid, shape doesn't match X,
                       or trying to overwrite an immutable layer.
        """
        # Validate layer name
        self._validate_name(key)

        # Validate shape matches X
        if value.shape != self.lazy_adata.shape:
            raise ValueError(
                f"Layer shape {value.shape} doesn't match X shape {self.lazy_adata.shape}"
            )

        # Check if layer already exists and is immutable
        if key in self.keys():
            if self._is_immutable(key):
                raise ValueError(
                    f"Layer '{key}' is immutable (converted from h5ad) and cannot be overwritten"
                )

        # Convert to materialized sparse matrix if needed
        from_lazy = isinstance(value, LazyExpressionMatrix)
        if from_lazy:
            value = value.compute()  # Materialize the lazy matrix

        # Ensure it's a sparse matrix
        import scipy.sparse

        if not scipy.sparse.issparse(value):
            # Convert dense to sparse
            value = scipy.sparse.csr_matrix(value)

        # Write immediately (eager write)
        self._write_layer_immediate(key, value, from_lazy=from_lazy)

    def _write_layer_immediate(
        self, layer_name: str, layer_matrix, from_lazy: bool = False
    ):
        """
        Write layer immediately to layers.lance (eager write).

        This method handles the immediate write of a layer to the layers.lance table.
        It's designed for small datasets (~10k cells) where the entire layer fits in memory.

        Args:
            layer_name: Name of the layer
            layer_matrix: Sparse matrix to write
            from_lazy: If True, the matrix came from LazyExpressionMatrix.compute(),
                      so coo.row and coo.col are already integer IDs
        """
        # Ensure layers table exists (create if needed)
        layers_path = self._slaf_array._join_path(
            self._slaf_array.slaf_path,
            self._slaf_array.config.get("tables", {}).get("layers", "layers.lance"),
        )

        # Convert sparse matrix to COO
        coo = layer_matrix.tocoo()

        # Get cell and gene integer IDs
        if from_lazy:
            # Matrix came from LazyExpressionMatrix.compute(), so row/col are already integer IDs
            cell_integer_ids = coo.row
            gene_integer_ids = coo.col
        else:
            # Matrix came from external source, need to map row/col indices to integer IDs
            obs_df = self.lazy_adata.slaf.obs
            var_df = self.lazy_adata.slaf.var
            cell_integer_ids = obs_df["cell_integer_id"].to_numpy()[coo.row]
            gene_integer_ids = var_df["gene_integer_id"].to_numpy()[coo.col]

        # Optimize dtype
        values, value_pa_type = self._optimize_dtype_for_layer(coo.data)

        # Create PyArrow table with the new layer column
        layer_table = pa.table(
            {
                "cell_integer_id": pa.array(cell_integer_ids, type=pa.uint32()),
                "gene_integer_id": pa.array(gene_integer_ids, type=pa.uint16()),
                layer_name: pa.array(values, type=value_pa_type),
            }
        )

        # Check if layers table exists
        if self._slaf_array.layers is None:
            # Create new layers table from expression table structure
            self._create_layers_table_with_layer(layer_table, layer_name, layers_path)
        else:
            # Update existing layers table
            self._update_layers_table_with_layer(layer_table, layer_name, layers_path)

        # Update config.json atomically
        self._update_config_layers_list([layer_name], add=True)

    def _optimize_dtype_for_layer(self, data: np.ndarray) -> tuple[np.ndarray, str]:
        """
        Optimize dtype for layer values.

        If float32 data contains only integers within uint16 range, convert to uint16.
        Otherwise, use float32.

        Args:
            data: Array of layer values

        Returns:
            Tuple of (optimized values array, value type string)
        """
        if len(data) == 0:
            return np.array([], dtype=np.float32), "float32"

        # Sample data to determine dtype
        sample_size = min(10000, len(data))
        sample_data = data[:sample_size]

        # Check if data is integer or float
        is_integer = np.issubdtype(sample_data.dtype, np.integer)
        max_value = np.max(data)
        min_value = np.min(data)

        if is_integer and max_value <= 65535 and min_value >= 0:
            return data.astype(np.uint16), "uint16"
        elif not is_integer:
            # Check if float data contains only integer values
            rounded_data = np.round(data)
            is_integer_values = np.allclose(data, rounded_data, rtol=1e-10)

            if is_integer_values and max_value <= 65535 and min_value >= 0:
                return rounded_data.astype(np.uint16), "uint16"
            else:
                return data.astype(np.float32), "float32"
        else:
            return data.astype(np.float32), "float32"

    def _get_pyarrow_type(self, value_type: str) -> pa.DataType:
        """
        Get PyArrow data type for a given value type string.

        Args:
            value_type: "uint16" or "float32"

        Returns:
            PyArrow data type
        """
        if value_type == "uint16":
            return pa.uint16()
        elif value_type == "float32":
            return pa.float32()
        else:
            raise ValueError(f"Unsupported value type: {value_type}")

    def _create_layers_table_with_layer(
        self, layer_table: pa.Table, layer_name: str, layers_path: str
    ):
        """Create new layers table with a single layer."""
        import lance

        # Get base structure from expression table (all cell-gene pairs)
        expression_df = (
            pl.scan_pyarrow_dataset(self._slaf_array.expression)
            .select(["cell_integer_id", "gene_integer_id"])
            .unique()
            .collect()
        )

        # Convert to PyArrow
        base_table = expression_df.to_arrow()

        # Join with new layer data
        layer_df_raw = pl.from_arrow(layer_table)
        base_df_raw = pl.from_arrow(base_table)
        assert isinstance(layer_df_raw, pl.DataFrame), (
            "Expected DataFrame from Arrow table"
        )
        assert isinstance(base_df_raw, pl.DataFrame), (
            "Expected DataFrame from Arrow table"
        )
        layer_df = layer_df_raw
        base_df = base_df_raw

        # Left join to add layer column (nullable for sparse data)
        combined_df = base_df.join(
            layer_df, on=["cell_integer_id", "gene_integer_id"], how="left"
        )

        # Write new layers table
        lance.write_dataset(
            combined_df.to_arrow(),
            layers_path,
            mode="create",
            max_rows_per_file=10000000,
        )

        # Reload layers dataset
        self._slaf_array.layers = lance.dataset(layers_path)

    def _update_layers_table_with_layer(
        self, layer_table: pa.Table, layer_name: str, layers_path: str
    ):
        """Update existing layers table with a new/updated layer using add_columns() with UDF."""
        import lance

        layers_dataset = self._slaf_array.layers

        # Check if layer column already exists
        schema = layers_dataset.schema
        column_names = [field.name for field in schema]

        # Drop the old column if it exists (using Lance native method - metadata-only, very fast)
        if layer_name in column_names:
            layers_dataset = layers_dataset.drop_columns([layer_name])
            # Reload from path after drop_columns to get the updated dataset
            layers_dataset = lance.dataset(layers_path)

        # Convert layer data to polars for efficient lookup in UDF
        layer_df = pl.from_arrow(layer_table)

        # Create UDF that returns just the new column data
        # Note: Pass function directly to add_columns(), not decorated
        def add_layer_column_udf(batch):
            """
            UDF to add layer column by joining batch with layer data.

            Receives batch with existing columns (cell_integer_id, gene_integer_id, etc.),
            returns RecordBatch with just the new layer column.

            This processes the dataset in batches, joining each batch with the
            new layer data to add the column efficiently without rewriting
            existing data.
            """
            # Convert batch to polars to access cell_integer_id and gene_integer_id
            batch_df = pl.from_arrow(batch)

            # Join with layer data (left join preserves all rows, nullable for sparse data)
            result_df = batch_df.join(
                layer_df, on=["cell_integer_id", "gene_integer_id"], how="left"
            )

            # Extract just the new layer column and convert to single array
            new_layer_chunked = result_df.select([layer_name]).to_arrow().column(0)
            new_layer_array = new_layer_chunked.combine_chunks()

            # Return RecordBatch with just the new column (names must match column name)
            return pa.RecordBatch.from_arrays(
                [new_layer_array],
                names=[layer_name],
            )

        # Add the column using add_columns() with UDF
        # This processes in batches and doesn't rewrite existing data
        layers_dataset.add_columns(add_layer_column_udf)

        # Reload layers dataset from path to ensure consistency
        self._slaf_array.layers = lance.dataset(layers_path)

    def _update_config_layers_list(self, layer_names: list[str], add: bool):
        """
        Update config.json to add or remove layers from available/mutable lists.

        Args:
            layer_names: List of layer names to add or remove
            add: If True, add layers; if False, remove layers
        """
        config_path = self._slaf_array._join_path(
            self._slaf_array.slaf_path, "config.json"
        )

        # Load existing config
        with self._slaf_array._open_file(config_path) as f:
            import json

            config = json.load(f)

        # Ensure layers config exists
        if "layers" not in config:
            config["layers"] = {"available": [], "immutable": [], "mutable": []}

        # Ensure tables config includes layers
        if "tables" not in config:
            config["tables"] = {}
        if "layers" not in config["tables"]:
            config["tables"]["layers"] = "layers.lance"

        layers_config = config["layers"]
        available = set(layers_config.get("available", []))
        immutable = set(layers_config.get("immutable", []))
        mutable = set(layers_config.get("mutable", []))

        if add:
            # Add layers to available and mutable (new layers are mutable)
            for layer_name in layer_names:
                available.add(layer_name)
                mutable.add(layer_name)
                # Remove from immutable if it was there (shouldn't happen, but be safe)
                immutable.discard(layer_name)
        else:
            # Remove layers from all lists
            for layer_name in layer_names:
                available.discard(layer_name)
                mutable.discard(layer_name)
                immutable.discard(layer_name)

        # Update config
        layers_config["available"] = sorted(available)
        layers_config["immutable"] = sorted(immutable)
        layers_config["mutable"] = sorted(mutable)

        # Save updated config
        # Note: this will not work for huggingface remote
        with self._slaf_array._open_file(config_path, "w") as f:
            json.dump(config, f, indent=2)

    def __delitem__(self, key: str):
        """
        Delete a layer (only if mutable).

        Args:
            key: Layer name to delete

        Raises:
            KeyError: If layer doesn't exist
            ValueError: If layer is immutable and cannot be deleted
        """
        import lance

        # Check if layer exists
        if key not in self.keys():
            raise KeyError(f"Layer '{key}' not found")

        # Check if layer is immutable
        if self._is_immutable(key):
            raise ValueError(
                f"Layer '{key}' is immutable (converted from h5ad) and cannot be deleted"
            )

        # Get layers path
        layers_path = self._slaf_array._join_path(
            self._slaf_array.slaf_path,
            self._slaf_array.config.get("tables", {}).get("layers", "layers.lance"),
        )

        # Drop the column using Lance's drop_columns() method (metadata-only, very fast)
        layers_dataset = self._slaf_array.layers
        layers_dataset = layers_dataset.drop_columns([key])

        # Reload layers dataset from path to ensure consistency
        self._slaf_array.layers = lance.dataset(layers_path)

        # Update config.json atomically
        self._update_config_layers_list([key], add=False)

Functions¶

`init(lazy_adata: LazyAnnData)` ¶

Initialize LazyLayersView with LazyAnnData.

Parameters:

Name	Type	Description	Default
`lazy_adata`	`LazyAnnData`	LazyAnnData instance containing the single-cell data.	required

Source code in slaf/integrations/anndata.py

def __init__(self, lazy_adata: "LazyAnnData"):
    """
    Initialize LazyLayersView with LazyAnnData.

    Args:
        lazy_adata: LazyAnnData instance containing the single-cell data.
    """
    self.lazy_adata = lazy_adata
    self._slaf_array = lazy_adata.slaf

`keys() -> list[str]` ¶

List all available layer names

Source code in slaf/integrations/anndata.py

def keys(self) -> list[str]:
    """List all available layer names"""
    # Fast path: read from config.json
    config_layers = self._get_layers_from_config()

    if config_layers:
        # Verify consistency with table if possible
        table_layers = self._get_layers_from_table()
        if table_layers and set(config_layers) != set(table_layers):
            # Log warning but prefer config
            import warnings

            warnings.warn(
                f"Layer names in config.json ({config_layers}) don't match "
                f"layers table ({table_layers}). Using config.json values.",
                UserWarning,
                stacklevel=2,
            )
        return config_layers

    # Fallback: query table
    return self._get_layers_from_table()

`LazyMetadataViewMixin` ¶

Bases: LazySparseMixin, LazyDictionaryViewMixin

Mixin class for metadata view operations (obs/var columns).

This mixin provides shared functionality for LazyObsView and LazyVarView, eliminating code duplication. It handles: - Dictionary-like interface (keys, getitem, setitem, delitem) - Column management (create, update, delete) - Config.json synchronization - Selector support - Immutability tracking

Required attributes (set by subclasses): - table_type: "obs" or "var" - table_name: "cells" or "genes" - id_column: "cell_integer_id" or "gene_integer_id" - lazy_adata: LazyAnnData instance - _slaf_array: SLAFArray instance - _shape: tuple[int, int] shape

Source code in slaf/integrations/anndata.py

class LazyMetadataViewMixin(LazySparseMixin, LazyDictionaryViewMixin):
    """
    Mixin class for metadata view operations (obs/var columns).

    This mixin provides shared functionality for LazyObsView and LazyVarView,
    eliminating code duplication. It handles:
    - Dictionary-like interface (keys, __getitem__, __setitem__, __delitem__)
    - Column management (create, update, delete)
    - Config.json synchronization
    - Selector support
    - Immutability tracking

    Required attributes (set by subclasses):
    - table_type: "obs" or "var"
    - table_name: "cells" or "genes"
    - id_column: "cell_integer_id" or "gene_integer_id"
    - lazy_adata: LazyAnnData instance
    - _slaf_array: SLAFArray instance
    - _shape: tuple[int, int] shape
    """

    # Type annotations for required attributes (set by subclasses)
    table_type: str
    table_name: str
    id_column: str
    lazy_adata: "LazyAnnData"
    _slaf_array: SLAFArray
    _shape: tuple[int, int]

    def _get_axis(self) -> int:
        """Get axis for this view (0 for obs/cells/obsm, 1 for var/genes/varm)"""
        return 0 if self.table_type in ("obs", "obsm") else 1

    def _get_current_selector(self) -> Any:
        """Get current selector from parent LazyAnnData"""
        # Map table_type to actual selector attribute names in LazyAnnData
        # obsm uses cell selector, varm uses gene selector
        selector_map = {
            "obs": "_cell_selector",
            "var": "_gene_selector",
            "obsm": "_cell_selector",
            "varm": "_gene_selector",
        }
        selector_attr = selector_map.get(self.table_type)
        if selector_attr:
            return getattr(self.lazy_adata, selector_attr, None)
        return None

    def _get_entity_count(self) -> int:
        """Get count of entities considering selectors"""
        selector = self._get_current_selector()
        axis = self._get_axis()
        if selector is None:
            return self._shape[axis]
        return self._get_selector_size(selector, axis)

    def _invalidate_metadata_cache(self):
        """
        Invalidate cached obs/var DataFrames when table structure changes.

        When we modify cells.lance or genes.lance (add/remove columns),
        the cached DataFrames in both LazyAnnData and SLAFArray become stale
        and need to be cleared so they reload from the updated tables.
        """
        # Invalidate LazyAnnData cache
        if hasattr(self.lazy_adata, "_obs"):
            self.lazy_adata._obs = None
        if hasattr(self.lazy_adata, "_var"):
            self.lazy_adata._var = None
        if hasattr(self.lazy_adata, "_cached_obs_names"):
            self.lazy_adata._cached_obs_names = None
        if hasattr(self.lazy_adata, "_cached_var_names"):
            self.lazy_adata._cached_var_names = None

        # Invalidate SLAFArray cache (which LazyAnnData.obs/var load from)
        # This ensures that when LazyAnnData.obs/var are accessed again,
        # they reload from the updated Lance tables
        if self.table_type in ("obs", "obsm"):
            # Modified cells.lance - invalidate SLAFArray._obs
            if hasattr(self._slaf_array, "_obs"):
                self._slaf_array._obs = None
            if hasattr(self._slaf_array, "_obs_columns"):
                self._slaf_array._obs_columns = None
            # Mark metadata as not loaded so it gets reloaded
            if hasattr(self._slaf_array, "_metadata_loaded"):
                self._slaf_array._metadata_loaded = False
        elif self.table_type in ("var", "varm"):
            # Modified genes.lance - invalidate SLAFArray._var
            if hasattr(self._slaf_array, "_var"):
                self._slaf_array._var = None
            if hasattr(self._slaf_array, "_var_columns"):
                self._slaf_array._var_columns = None
            # Mark metadata as not loaded so it gets reloaded
            if hasattr(self._slaf_array, "_metadata_loaded"):
                self._slaf_array._metadata_loaded = False

        # Invalidate view's cached DataFrame
        if hasattr(self, "_dataframe"):
            self._dataframe = None

    def _sql_condition_to_polars(self, sql_condition: str, id_column: str) -> pl.Expr:
        """Convert SQL WHERE condition to Polars expression"""
        if sql_condition == "TRUE":
            return pl.lit(True)
        if sql_condition == "FALSE":
            return pl.lit(False)

        # Handle range: "cell_integer_id >= 0 AND cell_integer_id < 100"
        if ">=" in sql_condition and "<" in sql_condition:
            parts = sql_condition.split(" AND ")
            ge_part = [p for p in parts if ">=" in p][0]
            lt_part = [p for p in parts if "<" in p][0]
            try:
                ge_value = int(ge_part.split(">=")[1].strip())
                lt_value = int(lt_part.split("<")[1].strip())
                return (pl.col(id_column) >= ge_value) & (pl.col(id_column) < lt_value)
            except ValueError:
                # If values are not integers, return False condition
                return pl.lit(False)

        # Handle IN clause: "cell_integer_id IN (0,1,2,3)"
        if " IN " in sql_condition:
            values_str = sql_condition.split(" IN ")[1].strip("()")
            # Filter out non-numeric values (like "False", "True", etc.)
            values = []
            for v in values_str.split(","):
                v = v.strip()
                try:
                    values.append(int(v))
                except ValueError:
                    # Skip non-integer values
                    continue
            if values:
                return pl.col(id_column).is_in(values)
            else:
                # If no valid values, return False condition
                return pl.lit(False)

        # Handle equality: "cell_integer_id = 5"
        if " = " in sql_condition:
            value = int(sql_condition.split(" = ")[1].strip())
            return pl.col(id_column) == value

        return pl.lit(True)  # Fallback: no filtering

    def _build_filtered_query(self, columns: list[str]) -> pl.LazyFrame:
        """Build filtered query for table with selectors"""
        table = getattr(self._slaf_array, self.table_name)
        query = pl.scan_pyarrow_dataset(table).select(columns)

        # Apply selector filtering using mixin utilities
        selector = self._get_current_selector()
        if selector is not None:
            # Convert selector to SQL condition, then to Polars filter
            axis = self._get_axis()
            entity_type = "cell" if self.table_type == "obs" else "gene"
            sql_condition = self._selector_to_sql_condition(
                selector, axis=axis, entity_type=entity_type
            )
            filter_expr = self._sql_condition_to_polars(sql_condition, self.id_column)
            query = query.filter(filter_expr)

        return query

    def _get_columns_from_config(self) -> list[str]:
        """Get column names from config.json (fast path)"""
        config = self._slaf_array.config
        if self.table_type in config and "available" in config[self.table_type]:
            return config[self.table_type]["available"]
        return []

    def _get_columns_from_table(self) -> list[str]:
        """Get column names by querying table (fallback)"""
        table = getattr(self._slaf_array, self.table_name, None)
        if table is None:
            return []

        try:
            schema = table.schema
            column_names = [field.name for field in schema]
            # Filter out system/internal columns:
            # - {id_column}: internal integer ID (not user-facing)
            # - {table_name}_start_index: Lance internal column (if present)
            system_cols = {self.id_column}
            # Handle both "cells" -> "cell_start_index" and "genes" -> "gene_start_index"
            if self.table_name == "cells":
                system_cols.add("cell_start_index")
            elif self.table_name == "genes":
                system_cols.add("gene_start_index")
            return [col for col in column_names if col not in system_cols]
        except Exception:
            return []

    def keys(self) -> list[str]:
        """List all available column/vector names"""
        # Route to vector or scalar method based on table_type
        if self.table_type in ("obsm", "varm"):
            return self._keys_vector()
        else:
            # Scalar column keys
            # Fast path: read from config.json
            config_columns = self._get_columns_from_config()

            if config_columns:
                # Verify consistency with table if possible
                table_columns = self._get_columns_from_table()
                if table_columns:
                    config_set = set(config_columns)
                    table_set = set(table_columns)

                    # Check for columns in config but not in table (real problem)
                    missing_in_table = config_set - table_set
                    if missing_in_table:
                        import warnings

                        warnings.warn(
                            f"Column names in config.json ({sorted(missing_in_table)}) "
                            f"not found in {self.table_name} table. These will be ignored.",
                            UserWarning,
                            stacklevel=2,
                        )

                    # Auto-sync: add columns from table that are missing in config
                    missing_in_config = table_set - config_set
                    if missing_in_config:
                        # Add missing columns to config (treat as immutable if they existed before)
                        self._sync_missing_columns_to_config(list(missing_in_config))
                        # Return updated config columns
                        config_columns = self._get_columns_from_config()

                return config_columns

            # Fallback: query table
            return self._get_columns_from_table()

    def __getitem__(self, key):
        """
        Get column/vector or DataFrame slice (AnnData-compatible DataFrame interface).

        - String key: Returns pandas Series (DataFrame-like behavior, matches AnnData)
        - Other keys (slice, list, etc.): Delegates to DataFrame indexing (DataFrame-like behavior)

        Args:
            key: Column name (str) or DataFrame indexer (slice, list, etc.)

        Returns:
            pandas Series if key is string, otherwise DataFrame slice
        """
        # Route to vector or scalar method based on table_type
        if self.table_type in ("obsm", "varm"):
            # For obsm/varm, only support string keys (dict-like)
            if not isinstance(key, str):
                raise TypeError(
                    f"{self.table_type} only supports string keys, got {type(key)}"
                )
            return self._get_vector_item(key)
        else:
            # For obs/var, use DataFrame-like access (AnnData-compatible)
            # Check if key exists first for better error messages
            if isinstance(key, str) and key not in self.keys():
                raise KeyError(f"Column '{key}' not found")
            # Always delegate to underlying DataFrame to get Series for string keys
            return self._get_dataframe().__getitem__(key)

    def _is_immutable(self, key: str) -> bool:
        """Check if column/vector key is immutable (converted from h5ad)"""
        # Route to vector or scalar method based on table_type
        if self.table_type in ("obsm", "varm"):
            return self._is_immutable_vector(key)
        else:
            # Scalar column immutability check
            config = self._slaf_array.config
            if self.table_type in config and "immutable" in config[self.table_type]:
                return key in config[self.table_type]["immutable"]
            return False

    def _optimize_dtype_for_column(
        self, data: np.ndarray
    ) -> tuple[np.ndarray, pa.DataType]:
        """
        Optimize dtype for column values.

        If float32 data contains only integers within uint16 range, convert to uint16.
        Otherwise, use float32.

        Args:
            data: Array of column values

        Returns:
            Tuple of (optimized values array, PyArrow data type)
        """
        if len(data) == 0:
            return np.array([], dtype=np.float32), pa.float32()

        # Handle string arrays
        if data.dtype.kind in [
            "U",
            "S",
            "O",
        ]:  # Unicode, byte string, or object (often strings)
            # Convert to string array and use PyArrow string type
            if data.dtype.kind == "O":
                # Object array - try to convert to string
                try:
                    data = np.array([str(x) for x in data], dtype="U")
                except (TypeError, ValueError):
                    # If conversion fails, keep as object
                    return data, pa.string()
            # Convert to UTF-8 string array
            return data.astype("U"), pa.string()

        # Sample data to determine dtype
        sample_size = min(10000, len(data))
        sample_data = data[:sample_size]

        # Check if data is integer or float
        is_integer = np.issubdtype(sample_data.dtype, np.integer)
        max_value = np.max(data)
        min_value = np.min(data)

        if is_integer and max_value <= 65535 and min_value >= 0:
            return data.astype(np.uint16), pa.uint16()
        elif not is_integer:
            # Check if float data contains only integer values
            rounded_data = np.round(data)
            is_integer_values = np.allclose(data, rounded_data, rtol=1e-10)

            if is_integer_values and max_value <= 65535 and min_value >= 0:
                return rounded_data.astype(np.uint16), pa.uint16()
            else:
                return data.astype(np.float32), pa.float32()
        else:
            return data.astype(np.float32), pa.float32()

    def __setitem__(self, key, value):
        """
        Create or update a column/vector or DataFrame slice (dual interface).

        - String key: Column assignment (dict-like behavior)
        - Other keys: DataFrame assignment (DataFrame-like behavior)

        Args:
            key: Column name (str) or DataFrame indexer
            value: numpy array, pandas Series, or DataFrame slice value
        """
        # Route to vector or scalar method based on table_type
        if self.table_type in ("obsm", "varm"):
            # For obsm/varm, only support string keys (dict-like)
            if not isinstance(key, str):
                raise TypeError(
                    f"{self.table_type} only supports string keys, got {type(key)}"
                )
            # Convert to numpy array if needed
            if isinstance(value, pd.Series):
                value = value.values
            value = np.asarray(value)
            # For vectors, value should be 2D (n_entities, n_dims)
            if len(value.shape) == 1:
                # If 1D, treat as single dimension vector
                value = value.reshape(-1, 1)
            self._set_vector_item(key, value)
        else:
            # For obs/var, support both dict-like and DataFrame-like assignment
            if isinstance(key, str):
                # Dict-like: column assignment
                self._set_column_item(key, value)
            else:
                # DataFrame-like: delegate to underlying DataFrame
                # Note: This will modify the DataFrame but won't persist to Lance
                # For now, we'll raise an error to guide users to use string keys for mutations
                raise NotImplementedError(
                    f"DataFrame-like assignment (e.g., obs[{key}] = ...) is not supported. "
                    f"Use column assignment (e.g., obs['{key}'] = ...) for mutations."
                )

    def _set_column_item(self, key: str, value: np.ndarray | pd.Series):
        """
        Create or update a column (eager write - immediate).

        Args:
            key: Column name (must be alphanumeric + underscore, non-empty)
            value: numpy array or pandas Series to assign.
                   Must have length matching entity count (considering selectors).

        Raises:
            ValueError: If column name is invalid, length doesn't match,
                       or trying to overwrite an immutable column.
        """
        # Validate column name
        self._validate_name(key)

        # Validate shape matches entity count (considering selectors)
        expected_count = self._get_entity_count()
        entity_name = self.table_type  # "obs" or "var"
        if len(value) != expected_count:
            raise ValueError(
                f"Column length {len(value)} doesn't match {entity_name} count {expected_count}"
            )

        # Check if column already exists and is immutable
        if key in self.keys() and self._is_immutable(key):
            raise ValueError(
                f"Column '{key}' is immutable (converted from h5ad) and cannot be overwritten"
            )

        # Convert to numpy array
        if isinstance(value, pd.Series):
            value = value.values
        value = np.asarray(value)

        # Optimize dtype
        values, value_pa_type = self._optimize_dtype_for_column(value)

        # Get integer IDs in order (respecting selectors)
        selector = self._get_current_selector()
        table = getattr(self._slaf_array, self.table_name)
        if selector is None:
            # No selector - use all entities
            df = table.to_table().to_pandas()
            integer_ids = df[self.id_column].values
        else:
            # Apply selector to get integer IDs
            query = self._build_filtered_query([self.id_column])
            df = query.collect().sort(self.id_column)
            integer_ids = df[self.id_column].to_numpy()

        # Determine ID column type (uint32 for cells, uint16 for genes)
        id_pa_type = pa.uint32() if self.table_type == "obs" else pa.uint16()

        # Create PyArrow table with the new column
        column_table = pa.table(
            {
                self.id_column: pa.array(integer_ids, type=id_pa_type),
                key: pa.array(values, type=value_pa_type),
            }
        )

        # Write to table
        table_path = self._slaf_array._join_path(
            self._slaf_array.slaf_path,
            self._slaf_array.config.get("tables", {}).get(
                self.table_name, f"{self.table_name}.lance"
            ),
        )

        if table is None:
            raise ValueError(f"{self.table_name}.lance table not found")

        # Update table (similar to layers update logic)
        self._update_table_with_column(column_table, key, table_path)

        # Update config.json atomically
        self._update_config_columns_list([key], add=True)

        # Reload config to ensure it's up-to-date (config is cached in SLAFArray)
        config_path = self._slaf_array._join_path(
            self._slaf_array.slaf_path, "config.json"
        )
        with self._slaf_array._open_file(config_path) as f:
            import json

            self._slaf_array.config = json.load(f)

        # Invalidate cached obs/var DataFrames since table structure changed
        self._invalidate_metadata_cache()

    def _update_table_with_column(
        self, column_table: pa.Table, column_name: str, table_path: str
    ):
        """Update existing table with a new/updated column using add_columns() with UDF."""
        import lance

        table_dataset = getattr(self._slaf_array, self.table_name)

        # Check if column already exists
        schema = table_dataset.schema
        column_names = [field.name for field in schema]

        # Drop the old column if it exists (using Lance native method - metadata-only, very fast)
        if column_name in column_names:
            table_dataset = table_dataset.drop_columns([column_name])
            # Reload from path after drop_columns to get the updated dataset
            table_dataset = lance.dataset(table_path)

        # Convert column data to polars for efficient lookup in UDF
        column_df = pl.from_arrow(column_table)

        # Create UDF that returns just the new column data
        def add_column_udf(batch):
            """
            UDF to add column by joining batch with column data.

            Receives batch with existing columns (cell_integer_id, etc.),
            returns RecordBatch with just the new column.
            """
            # Convert batch to polars
            batch_df = pl.from_arrow(batch)

            # Join with column data (left join preserves all rows)
            result_df = batch_df.join(column_df, on=[self.id_column], how="left")

            # Extract just the new column and convert to single array
            new_column_chunked = result_df.select([column_name]).to_arrow().column(0)
            new_column_array = new_column_chunked.combine_chunks()

            # Return RecordBatch with just the new column
            return pa.RecordBatch.from_arrays(
                [new_column_array],
                names=[column_name],
            )

        # Add the column using add_columns() with UDF
        table_dataset.add_columns(add_column_udf)

        # Reload table dataset from path to ensure consistency
        setattr(self._slaf_array, self.table_name, lance.dataset(table_path))

    def _update_config_columns_list(self, column_names: list[str], add: bool):
        """
        Update config.json to add or remove columns from available/mutable lists.

        Args:
            column_names: List of column names to add or remove
            add: If True, add columns; if False, remove columns
        """
        config_path = self._slaf_array._join_path(
            self._slaf_array.slaf_path, "config.json"
        )

        # Load existing config
        with self._slaf_array._open_file(config_path) as f:
            import json

            config = json.load(f)

        # Ensure table config exists
        if self.table_type not in config:
            config[self.table_type] = {"available": [], "immutable": [], "mutable": []}

        table_config = config[self.table_type]
        available = set(table_config.get("available", []))
        immutable = set(table_config.get("immutable", []))
        mutable = set(table_config.get("mutable", []))

        if add:
            # Add columns to available and mutable (new columns are mutable)
            for column_name in column_names:
                available.add(column_name)
                mutable.add(column_name)
                # Remove from immutable if it was there (shouldn't happen, but be safe)
                immutable.discard(column_name)
        else:
            # Remove columns from all lists
            for column_name in column_names:
                available.discard(column_name)
                mutable.discard(column_name)
                immutable.discard(column_name)

        # Update config
        table_config["available"] = sorted(available)
        table_config["immutable"] = sorted(immutable)
        table_config["mutable"] = sorted(mutable)

        # Save updated config
        with self._slaf_array._open_file(config_path, "w") as f:
            json.dump(config, f, indent=2)

    def _sync_missing_columns_to_config(self, column_names: list[str]):
        """
        Sync missing columns from table to config.json.
        These columns are treated as immutable (they existed before Phase 6.5).

        Args:
            column_names: List of column names to add to config
        """
        if not column_names:
            return

        config_path = self._slaf_array._join_path(
            self._slaf_array.slaf_path, "config.json"
        )

        # Load existing config
        with self._slaf_array._open_file(config_path) as f:
            import json

            config = json.load(f)

        # Ensure table config exists
        if self.table_type not in config:
            config[self.table_type] = {"available": [], "immutable": [], "mutable": []}

        table_config = config[self.table_type]
        available = set(table_config.get("available", []))
        immutable = set(table_config.get("immutable", []))
        mutable = set(table_config.get("mutable", []))

        # Add missing columns to available and immutable (they existed before)
        for column_name in column_names:
            available.add(column_name)
            immutable.add(column_name)
            # Remove from mutable if it was there (shouldn't happen, but be safe)
            mutable.discard(column_name)

        # Update config
        table_config["available"] = sorted(available)
        table_config["immutable"] = sorted(immutable)
        table_config["mutable"] = sorted(mutable)

        # Save updated config
        with self._slaf_array._open_file(config_path, "w") as f:
            json.dump(config, f, indent=2)

    def __delitem__(self, key: str):
        """
        Delete a column/vector (only if mutable).

        Args:
            key: Column/vector name to delete

        Raises:
            KeyError: If column/vector doesn't exist
            ValueError: If column/vector is immutable and cannot be deleted
        """
        # Route to vector or scalar method based on table_type
        if self.table_type in ("obsm", "varm"):
            self._del_vector_item(key)
        else:
            # Scalar column deletion
            import lance

            # Check if column exists
            if key not in self.keys():
                raise KeyError(f"Column '{key}' not found")

            # Check if column is immutable
            if self._is_immutable(key):
                raise ValueError(
                    f"Column '{key}' is immutable (converted from h5ad) and cannot be deleted"
                )

            # Get table path
            table_path = self._slaf_array._join_path(
                self._slaf_array.slaf_path,
                self._slaf_array.config.get("tables", {}).get(
                    self.table_name, f"{self.table_name}.lance"
                ),
            )

            # Drop the column using Lance's drop_columns() method (metadata-only, very fast)
            table_dataset = getattr(self._slaf_array, self.table_name)
            table_dataset = table_dataset.drop_columns([key])

            # Reload table dataset from path to ensure consistency
            setattr(self._slaf_array, self.table_name, lance.dataset(table_path))

            # Update config.json atomically
            self._update_config_columns_list([key], add=False)

            # Reload config to ensure it's up-to-date (config is cached in SLAFArray)
            config_path = self._slaf_array._join_path(
                self._slaf_array.slaf_path, "config.json"
            )
            with self._slaf_array._open_file(config_path) as f:
                import json

                self._slaf_array.config = json.load(f)

            # Invalidate cached obs/var DataFrames since table structure changed
            self._invalidate_metadata_cache()

    # ==================== Vector-specific methods (for obsm/varm) ====================

    def _detect_vector_columns(self) -> dict[str, int]:
        """
        Detect FixedSizeListArray columns from schema and return key -> dimension mapping.

        Returns:
            Dictionary mapping vector key names to their dimensions
        """
        table = getattr(self._slaf_array, self.table_name, None)
        if table is None:
            return {}

        vector_columns = {}
        schema = table.schema

        for field in schema:
            # Check if field is a FixedSizeListArray (vector type)
            if isinstance(field.type, pa.FixedSizeListType):
                # Use column name directly as the key
                key = field.name
                # Get dimension from FixedSizeListType
                n_dims = field.type.list_size
                vector_columns[key] = n_dims

        return vector_columns

    def _get_vector_item(self, key: str) -> np.ndarray:
        """Retrieve multi-dimensional array (respects selectors from parent)"""
        if key not in self.keys():
            raise KeyError(f"{self.table_type} key '{key}' not found")

        # Build query for the vector column (using mixin for filtering)
        query = self._build_filtered_query([self.id_column, key])
        df = query.collect()

        # Sort by integer ID
        df = df.sort(self.id_column)

        # Extract vector column and convert to numpy array
        # FixedSizeListArray columns are stored as lists/arrays
        vector_data = df[key].to_numpy()

        # Convert list of arrays to 2D numpy array
        if len(vector_data) > 0 and isinstance(vector_data[0], list | np.ndarray):
            return np.array([np.array(v) for v in vector_data])
        else:
            # Already in correct format
            return np.asarray(vector_data)

    def _set_vector_item(self, key: str, value: np.ndarray):
        """Store multi-dimensional array as FixedSizeListArray column"""
        import lance
        import pyarrow as pa

        # Validate shape using mixin utilities (respects selectors)
        expected_count = self._get_entity_count()
        if value.shape[0] != expected_count:
            raise ValueError(
                f"Array first dimension {value.shape[0]} doesn't match {self.table_type} count {expected_count}"
            )

        # Validate key name
        self._validate_name(key)

        # Check immutability
        if key in self.keys() and self._is_immutable(key):
            raise ValueError(
                f"{self.table_type} key '{key}' is immutable and cannot be overwritten"
            )

        # Convert to numpy array
        value = np.asarray(value)
        n_dims = value.shape[1] if len(value.shape) > 1 else 1

        # Get integer IDs in order (respecting selectors)
        selector = self._get_current_selector()
        table = getattr(self._slaf_array, self.table_name)
        if selector is None:
            # No selector - use all entities
            df = table.to_table().to_pandas()
            integer_ids = df[self.id_column].values
        else:
            # Apply selector to get integer IDs
            query = self._build_filtered_query([self.id_column])
            df = query.collect().sort(self.id_column)
            integer_ids = df[self.id_column].to_numpy()

        # Create FixedSizeListArray directly from numpy array
        # Determine value dtype (use float32 for embeddings)
        value_dtype = pa.float32() if value.dtype.kind == "f" else pa.float32()

        # Flatten the 2D array and create FixedSizeListArray
        # FixedSizeListArray.from_arrays takes a flat array and list_size
        flat_values = pa.array(value.flatten(), type=value_dtype)
        vector_array = pa.FixedSizeListArray.from_arrays(flat_values, n_dims)

        # If overwriting, drop old column first
        if key in self.keys():
            table_dataset = getattr(self._slaf_array, self.table_name)
            table_dataset = table_dataset.drop_columns([key])
            table_path = self._slaf_array._join_path(
                self._slaf_array.slaf_path,
                self._slaf_array.config.get("tables", {}).get(
                    self.table_name, f"{self.table_name}.lance"
                ),
            )
            setattr(self._slaf_array, self.table_name, lance.dataset(table_path))

        # Create table with id_column and vector column
        table_path = self._slaf_array._join_path(
            self._slaf_array.slaf_path,
            self._slaf_array.config.get("tables", {}).get(
                self.table_name, f"{self.table_name}.lance"
            ),
        )

        # Determine ID column type (uint32 for cells, uint16 for genes)
        id_pa_type = pa.uint32() if self.table_type == "obsm" else pa.uint16()

        # Create table with integer IDs and vector column
        column_table = pa.table(
            {
                self.id_column: pa.array(integer_ids, type=id_pa_type),
                key: vector_array,
            }
        )

        # Update table
        self._update_table_with_column(column_table, key, table_path)

        # Update config.json
        self._update_config_vector_list([key], add=True, n_dims=n_dims)

        # Reload config to ensure it's up-to-date (config is cached in SLAFArray)
        config_path = self._slaf_array._join_path(
            self._slaf_array.slaf_path, "config.json"
        )
        with self._slaf_array._open_file(config_path) as f:
            import json

            self._slaf_array.config = json.load(f)

        # Invalidate cached obs/var DataFrames since table structure changed
        self._invalidate_metadata_cache()

    def _del_vector_item(self, key: str):
        """Delete vector key (drops the vector column)"""
        import lance

        if key not in self.keys():
            raise KeyError(f"{self.table_type} key '{key}' not found")

        if self._is_immutable(key):
            raise ValueError(
                f"{self.table_type} key '{key}' is immutable and cannot be deleted"
            )

        # Drop the vector column
        table_dataset = getattr(self._slaf_array, self.table_name)
        table_dataset = table_dataset.drop_columns([key])

        # Reload dataset
        table_path = self._slaf_array._join_path(
            self._slaf_array.slaf_path,
            self._slaf_array.config.get("tables", {}).get(
                self.table_name, f"{self.table_name}.lance"
            ),
        )
        setattr(self._slaf_array, self.table_name, lance.dataset(table_path))

        # Update config.json
        self._update_config_vector_list([key], add=False)

        # Reload config to ensure it's up-to-date (config is cached in SLAFArray)
        config_path = self._slaf_array._join_path(
            self._slaf_array.slaf_path, "config.json"
        )
        with self._slaf_array._open_file(config_path) as f:
            import json

            self._slaf_array.config = json.load(f)

        # Invalidate cached obs/var DataFrames since table structure changed
        self._invalidate_metadata_cache()

    def _keys_vector(self) -> list[str]:
        """List all available vector keys by detecting FixedSizeListArray columns"""
        # Fast path: read from config.json
        config = self._slaf_array.config
        if self.table_type in config and "available" in config[self.table_type]:
            config_keys = config[self.table_type]["available"]
            # Verify against schema (auto-sync if needed)
            schema_keys = list(self._detect_vector_columns().keys())
            if set(config_keys) != set(schema_keys):
                # Auto-sync: update config with schema keys
                missing_in_config = set(schema_keys) - set(config_keys)
                if missing_in_config:
                    # Add missing keys to config (treat as immutable if they existed before)
                    self._update_config_vector_list(list(missing_in_config), add=True)
                    # Re-read config
                    config = self._slaf_array.config
                    return config.get(self.table_type, {}).get("available", [])
            return config_keys

        # Fallback: detect from schema
        return list(self._detect_vector_columns().keys())

    def _is_immutable_vector(self, key: str) -> bool:
        """Check if vector key is immutable"""
        config = self._slaf_array.config
        if self.table_type in config and "immutable" in config[self.table_type]:
            return key in config[self.table_type]["immutable"]
        return False

    def _update_config_vector_list(
        self, keys: list[str], add: bool, n_dims: int | None = None
    ):
        """Update config.json to add or remove vector keys (unified for obsm/varm)"""
        config_path = self._slaf_array._join_path(
            self._slaf_array.slaf_path, "config.json"
        )

        # Load existing config
        with self._slaf_array._open_file(config_path) as f:
            import json

            config = json.load(f)

        # Ensure vector config exists
        if self.table_type not in config:
            config[self.table_type] = {
                "available": [],
                "immutable": [],
                "mutable": [],
                "dimensions": {},
            }

        vector_config = config[self.table_type]
        available = set(vector_config.get("available", []))
        immutable = set(vector_config.get("immutable", []))
        mutable = set(vector_config.get("mutable", []))
        dimensions = vector_config.get("dimensions", {})

        if add:
            # Add keys to available and mutable (new keys are mutable)
            for key in keys:
                available.add(key)
                mutable.add(key)
                immutable.discard(key)
                if n_dims is not None:
                    dimensions[key] = n_dims
        else:
            # Remove keys from all lists
            for key in keys:
                available.discard(key)
                mutable.discard(key)
                immutable.discard(key)
                dimensions.pop(key, None)

        # Update config
        vector_config["available"] = sorted(available)
        vector_config["immutable"] = sorted(immutable)
        vector_config["mutable"] = sorted(mutable)
        vector_config["dimensions"] = dimensions

        # Save updated config
        with self._slaf_array._open_file(config_path, "w") as f:
            json.dump(config, f, indent=2)

Functions¶

`keys() -> list[str]` ¶

List all available column/vector names

Source code in slaf/integrations/anndata.py

def keys(self) -> list[str]:
    """List all available column/vector names"""
    # Route to vector or scalar method based on table_type
    if self.table_type in ("obsm", "varm"):
        return self._keys_vector()
    else:
        # Scalar column keys
        # Fast path: read from config.json
        config_columns = self._get_columns_from_config()

        if config_columns:
            # Verify consistency with table if possible
            table_columns = self._get_columns_from_table()
            if table_columns:
                config_set = set(config_columns)
                table_set = set(table_columns)

                # Check for columns in config but not in table (real problem)
                missing_in_table = config_set - table_set
                if missing_in_table:
                    import warnings

                    warnings.warn(
                        f"Column names in config.json ({sorted(missing_in_table)}) "
                        f"not found in {self.table_name} table. These will be ignored.",
                        UserWarning,
                        stacklevel=2,
                    )

                # Auto-sync: add columns from table that are missing in config
                missing_in_config = table_set - config_set
                if missing_in_config:
                    # Add missing columns to config (treat as immutable if they existed before)
                    self._sync_missing_columns_to_config(list(missing_in_config))
                    # Return updated config columns
                    config_columns = self._get_columns_from_config()

            return config_columns

        # Fallback: query table
        return self._get_columns_from_table()

`LazyObsView` ¶

Bases: LazyMetadataViewMixin

Dual-interface view of obs columns: DataFrame-like and dictionary-like.

LazyObsView provides both DataFrame-like and dictionary-like interfaces for accessing and mutating cell metadata columns stored in the cells.lance table.

Key Features

DataFrame-like interface: obs.columns, obs.head(), obs[slice] (AnnData-compatible)
AnnData-compatible access: obs["col"] returns pd.Series (not np.ndarray)
Dictionary-like interface: "col" in obs, len(obs), obs.keys()
Lazy evaluation: columns are accessed on-demand
Selector support: respects cell selectors from parent LazyAnnData
Immutability: prevents deletion/modification of converted columns
Config.json consistency: reads from config for fast column discovery

Examples:

>>> # DataFrame-like access (AnnData-compatible)
>>> slaf_array = SLAFArray("data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> df = adata.obs  # Returns DataFrame-like view
>>> print(df.columns)  # DataFrame columns
>>> print(df.head())  # DataFrame methods work

>>> # AnnData-compatible column access (returns Series)
>>> cluster = adata.obs["cluster"]  # Returns pd.Series (AnnData-compatible)
>>> print(type(cluster))
<class 'pandas.core.series.Series'>

>>> # Create a new column
>>> adata.obs["new_cluster"] = new_cluster_labels
>>> assert "new_cluster" in adata.obs

>>> # List available columns
>>> print(list(adata.obs.keys()))
['cell_id', 'total_counts', 'cluster', 'new_cluster']

Source code in slaf/integrations/anndata.py

class LazyObsView(LazyMetadataViewMixin):
    """
    Dual-interface view of obs columns: DataFrame-like and dictionary-like.

    LazyObsView provides both DataFrame-like and dictionary-like interfaces for accessing
    and mutating cell metadata columns stored in the cells.lance table.

    Key Features:
        - DataFrame-like interface: obs.columns, obs.head(), obs[slice] (AnnData-compatible)
        - AnnData-compatible access: obs["col"] returns pd.Series (not np.ndarray)
        - Dictionary-like interface: "col" in obs, len(obs), obs.keys()
        - Lazy evaluation: columns are accessed on-demand
        - Selector support: respects cell selectors from parent LazyAnnData
        - Immutability: prevents deletion/modification of converted columns
        - Config.json consistency: reads from config for fast column discovery

    Examples:
        >>> # DataFrame-like access (AnnData-compatible)
        >>> slaf_array = SLAFArray("data.slaf")
        >>> adata = LazyAnnData(slaf_array)
        >>> df = adata.obs  # Returns DataFrame-like view
        >>> print(df.columns)  # DataFrame columns
        >>> print(df.head())  # DataFrame methods work

        >>> # AnnData-compatible column access (returns Series)
        >>> cluster = adata.obs["cluster"]  # Returns pd.Series (AnnData-compatible)
        >>> print(type(cluster))
        <class 'pandas.core.series.Series'>

        >>> # Create a new column
        >>> adata.obs["new_cluster"] = new_cluster_labels
        >>> assert "new_cluster" in adata.obs

        >>> # List available columns
        >>> print(list(adata.obs.keys()))
        ['cell_id', 'total_counts', 'cluster', 'new_cluster']
    """

    def __init__(self, lazy_adata: "LazyAnnData"):
        """
        Initialize LazyObsView with LazyAnnData.

        Args:
            lazy_adata: LazyAnnData instance containing the single-cell data.
        """
        super().__init__()
        self.lazy_adata = lazy_adata
        self._slaf_array = lazy_adata.slaf
        # Required by LazySparseMixin
        self.slaf_array = lazy_adata.slaf
        self._shape = lazy_adata.shape  # (n_cells, n_genes)
        self.table_name = "cells"
        self.id_column = "cell_integer_id"
        self.table_type = "obs"
        self._dataframe: pd.DataFrame | None = None  # Cached DataFrame

    @property
    def shape(self) -> tuple[int, int]:
        """Required by LazySparseMixin - uses parent's shape"""
        return self._shape

    def __len__(self) -> int:
        """
        Return number of rows (DataFrame-like behavior).

        For DataFrame compatibility, len(obs) should return the number of rows,
        not the number of columns (which is what the dictionary interface would return).
        """
        return len(self._get_dataframe())

    def _get_dataframe(self) -> pd.DataFrame:
        """
        Get underlying DataFrame (lazy-loaded, respects selectors).

        Returns:
            pandas DataFrame with all columns from cells.lance (excluding vector columns)
        """
        # If parent has filtered_obs function, use it instead
        if (
            hasattr(self.lazy_adata, "_filtered_obs")
            and self.lazy_adata._filtered_obs is not None
        ):
            return self.lazy_adata._filtered_obs()

        if self._dataframe is None:
            # Get all column names from table schema
            table = getattr(self._slaf_array, self.table_name)
            schema = table.schema
            all_columns = [field.name for field in schema]

            # Build DataFrame from cells.lance with selectors
            query = self._build_filtered_query(all_columns)
            df = query.collect()

            # Sort by integer ID to match order
            df = df.sort(self.id_column)

            # Convert to pandas DataFrame
            obs_df = df.to_pandas()

            # Drop system columns
            if self.id_column in obs_df.columns:
                obs_df = obs_df.drop(columns=[self.id_column])

            # Filter out vector columns (obsm) - these are FixedSizeListArray columns
            cells_table = getattr(self._slaf_array, self.table_name, None)
            if cells_table is not None:
                schema = cells_table.schema
                vector_column_names = {
                    field.name
                    for field in schema
                    if isinstance(field.type, pa.FixedSizeListType)
                }
                columns_to_drop = [
                    col for col in obs_df.columns if col in vector_column_names
                ]
                if columns_to_drop:
                    obs_df = obs_df.drop(columns=columns_to_drop)

            # Set cell_id as index if present
            if "cell_id" in obs_df.columns:
                obs_df = obs_df.set_index("cell_id")
                obs_df.index.name = "cell_id"

            self._dataframe = obs_df

        return self._dataframe

    def __getattr__(self, name: str):
        """
        Delegate DataFrame attributes to underlying DataFrame.

        This allows obs to behave like a DataFrame when accessed directly,
        e.g., obs.columns, obs.head(), obs.shape, etc.
        """
        # Don't delegate special methods or our own methods
        if name.startswith("_") or name in dir(self):
            raise AttributeError(
                f"'{type(self).__name__}' object has no attribute '{name}'"
            )

        # Delegate to underlying DataFrame
        try:
            return getattr(self._get_dataframe(), name)
        except AttributeError as err:
            raise AttributeError(
                f"'{type(self).__name__}' object has no attribute '{name}'"
            ) from err

Attributes¶

`shape: tuple[int, int]` `property` ¶

Required by LazySparseMixin - uses parent's shape

Functions¶

`init(lazy_adata: LazyAnnData)` ¶

Initialize LazyObsView with LazyAnnData.

Parameters:

Name	Type	Description	Default
`lazy_adata`	`LazyAnnData`	LazyAnnData instance containing the single-cell data.	required

Source code in slaf/integrations/anndata.py

def __init__(self, lazy_adata: "LazyAnnData"):
    """
    Initialize LazyObsView with LazyAnnData.

    Args:
        lazy_adata: LazyAnnData instance containing the single-cell data.
    """
    super().__init__()
    self.lazy_adata = lazy_adata
    self._slaf_array = lazy_adata.slaf
    # Required by LazySparseMixin
    self.slaf_array = lazy_adata.slaf
    self._shape = lazy_adata.shape  # (n_cells, n_genes)
    self.table_name = "cells"
    self.id_column = "cell_integer_id"
    self.table_type = "obs"
    self._dataframe: pd.DataFrame | None = None  # Cached DataFrame

`LazyVarView` ¶

Bases: LazyMetadataViewMixin

Dual-interface view of var columns: DataFrame-like and dictionary-like.

LazyVarView provides both DataFrame-like and dictionary-like interfaces for accessing and mutating gene metadata columns stored in the genes.lance table.

Key Features

DataFrame-like interface: var.columns, var.head(), var[slice] (AnnData-compatible)
AnnData-compatible access: var["col"] returns pd.Series (not np.ndarray)
Dictionary-like interface: "col" in var, len(var), var.keys()
Lazy evaluation: columns are accessed on-demand
Selector support: respects gene selectors from parent LazyAnnData
Immutability: prevents deletion/modification of converted columns
Config.json consistency: reads from config for fast column discovery

Examples:

>>> # DataFrame-like access (AnnData-compatible)
>>> slaf_array = SLAFArray("data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> df = adata.var  # Returns DataFrame-like view
>>> print(df.columns)  # DataFrame columns
>>> print(df.head())  # DataFrame methods work

>>> # AnnData-compatible column access (returns Series)
>>> hvg = adata.var["highly_variable"]  # Returns pd.Series (AnnData-compatible)
>>> print(type(hvg))
<class 'pandas.core.series.Series'>

>>> # Create a new column
>>> adata.var["new_annotation"] = new_annotations
>>> assert "new_annotation" in adata.var

Source code in slaf/integrations/anndata.py

class LazyVarView(LazyMetadataViewMixin):
    """
    Dual-interface view of var columns: DataFrame-like and dictionary-like.

    LazyVarView provides both DataFrame-like and dictionary-like interfaces for accessing
    and mutating gene metadata columns stored in the genes.lance table.

    Key Features:
        - DataFrame-like interface: var.columns, var.head(), var[slice] (AnnData-compatible)
        - AnnData-compatible access: var["col"] returns pd.Series (not np.ndarray)
        - Dictionary-like interface: "col" in var, len(var), var.keys()
        - Lazy evaluation: columns are accessed on-demand
        - Selector support: respects gene selectors from parent LazyAnnData
        - Immutability: prevents deletion/modification of converted columns
        - Config.json consistency: reads from config for fast column discovery

    Examples:
        >>> # DataFrame-like access (AnnData-compatible)
        >>> slaf_array = SLAFArray("data.slaf")
        >>> adata = LazyAnnData(slaf_array)
        >>> df = adata.var  # Returns DataFrame-like view
        >>> print(df.columns)  # DataFrame columns
        >>> print(df.head())  # DataFrame methods work

        >>> # AnnData-compatible column access (returns Series)
        >>> hvg = adata.var["highly_variable"]  # Returns pd.Series (AnnData-compatible)
        >>> print(type(hvg))
        <class 'pandas.core.series.Series'>

        >>> # Create a new column
        >>> adata.var["new_annotation"] = new_annotations
        >>> assert "new_annotation" in adata.var
    """

    def __init__(self, lazy_adata: "LazyAnnData"):
        """
        Initialize LazyVarView with LazyAnnData.

        Args:
            lazy_adata: LazyAnnData instance containing the single-cell data.
        """
        super().__init__()
        self.lazy_adata = lazy_adata
        self._slaf_array = lazy_adata.slaf
        # Required by LazySparseMixin
        self.slaf_array = lazy_adata.slaf
        self._shape = lazy_adata.shape  # (n_cells, n_genes)
        self.table_name = "genes"
        self.id_column = "gene_integer_id"
        self.table_type = "var"
        self._dataframe: pd.DataFrame | None = None  # Cached DataFrame

    @property
    def shape(self) -> tuple[int, int]:
        """Required by LazySparseMixin - uses parent's shape"""
        return self._shape

    def __len__(self) -> int:
        """
        Return number of rows (DataFrame-like behavior).

        For DataFrame compatibility, len(var) should return the number of rows,
        not the number of columns (which is what the dictionary interface would return).
        """
        return len(self._get_dataframe())

    def _get_dataframe(self) -> pd.DataFrame:
        """
        Get underlying DataFrame (lazy-loaded, respects selectors).

        Returns:
            pandas DataFrame with all columns from genes.lance (excluding vector columns)
        """
        # If parent has filtered_var function, use it instead
        if (
            hasattr(self.lazy_adata, "_filtered_var")
            and self.lazy_adata._filtered_var is not None
        ):
            return self.lazy_adata._filtered_var()

        if self._dataframe is None:
            # Get all column names from table schema
            table = getattr(self._slaf_array, self.table_name)
            schema = table.schema
            all_columns = [field.name for field in schema]

            # Build DataFrame from genes.lance with selectors
            query = self._build_filtered_query(all_columns)
            df = query.collect()

            # Sort by integer ID to match order
            df = df.sort(self.id_column)

            # Convert to pandas DataFrame
            var_df = df.to_pandas()

            # Drop system columns
            if self.id_column in var_df.columns:
                var_df = var_df.drop(columns=[self.id_column])

            # Filter out vector columns (varm) - these are FixedSizeListArray columns
            genes_table = getattr(self._slaf_array, self.table_name, None)
            if genes_table is not None:
                schema = genes_table.schema
                vector_column_names = {
                    field.name
                    for field in schema
                    if isinstance(field.type, pa.FixedSizeListType)
                }
                columns_to_drop = [
                    col for col in var_df.columns if col in vector_column_names
                ]
                if columns_to_drop:
                    var_df = var_df.drop(columns=columns_to_drop)

            # Set gene_id as index if present
            if "gene_id" in var_df.columns:
                var_df = var_df.set_index("gene_id")
                var_df.index.name = "gene_id"

            self._dataframe = var_df

        return self._dataframe

    def __getattr__(self, name: str):
        """
        Delegate DataFrame attributes to underlying DataFrame.

        This allows var to behave like a DataFrame when accessed directly,
        e.g., var.columns, var.head(), var.shape, etc.
        """
        # Don't delegate special methods or our own methods
        if name.startswith("_") or name in dir(self):
            raise AttributeError(
                f"'{type(self).__name__}' object has no attribute '{name}'"
            )

        # Delegate to underlying DataFrame
        try:
            return getattr(self._get_dataframe(), name)
        except AttributeError as err:
            raise AttributeError(
                f"'{type(self).__name__}' object has no attribute '{name}'"
            ) from err

Attributes¶

`shape: tuple[int, int]` `property` ¶

Required by LazySparseMixin - uses parent's shape

Functions¶

`init(lazy_adata: LazyAnnData)` ¶

Initialize LazyVarView with LazyAnnData.

Parameters:

Name	Type	Description	Default
`lazy_adata`	`LazyAnnData`	LazyAnnData instance containing the single-cell data.	required

Source code in slaf/integrations/anndata.py

def __init__(self, lazy_adata: "LazyAnnData"):
    """
    Initialize LazyVarView with LazyAnnData.

    Args:
        lazy_adata: LazyAnnData instance containing the single-cell data.
    """
    super().__init__()
    self.lazy_adata = lazy_adata
    self._slaf_array = lazy_adata.slaf
    # Required by LazySparseMixin
    self.slaf_array = lazy_adata.slaf
    self._shape = lazy_adata.shape  # (n_cells, n_genes)
    self.table_name = "genes"
    self.id_column = "gene_integer_id"
    self.table_type = "var"
    self._dataframe: pd.DataFrame | None = None  # Cached DataFrame

`LazyObsmView` ¶

Bases: LazyMetadataViewMixin

Dictionary-like view of obsm (multi-dimensional obs annotations).

LazyObsmView provides a dictionary-like interface for accessing and mutating multi-dimensional cell annotations (e.g., UMAP, PCA embeddings) stored as separate columns in the cells.lance table. Each key maps to a 2D numpy array.

Key Features

Dictionary-like interface: obsm["X_umap"], "X_umap" in obsm, len(obsm)
Multi-dimensional arrays: stored as FixedSizeListArray columns (native Lance vector type)
Schema-based detection: automatically detects vector columns from Lance schema
Selector support: respects cell selectors from parent LazyAnnData
Immutability: prevents deletion/modification of converted embeddings
Config.json consistency: reads from config for fast key discovery

Examples:

>>> # Access an embedding
>>> slaf_array = SLAFArray("data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> umap = adata.obsm["X_umap"]
>>> print(f"UMAP shape: {umap.shape}")
UMAP shape: (1000, 2)

>>> # Create a new embedding
>>> adata.obsm["X_pca"] = pca_coords  # shape: (1000, 50)
>>> assert "X_pca" in adata.obsm

Source code in slaf/integrations/anndata.py

class LazyObsmView(LazyMetadataViewMixin):
    """
    Dictionary-like view of obsm (multi-dimensional obs annotations).

    LazyObsmView provides a dictionary-like interface for accessing and mutating
    multi-dimensional cell annotations (e.g., UMAP, PCA embeddings) stored as
    separate columns in the cells.lance table. Each key maps to a 2D numpy array.

    Key Features:
        - Dictionary-like interface: obsm["X_umap"], "X_umap" in obsm, len(obsm)
        - Multi-dimensional arrays: stored as FixedSizeListArray columns (native Lance vector type)
        - Schema-based detection: automatically detects vector columns from Lance schema
        - Selector support: respects cell selectors from parent LazyAnnData
        - Immutability: prevents deletion/modification of converted embeddings
        - Config.json consistency: reads from config for fast key discovery

    Examples:
        >>> # Access an embedding
        >>> slaf_array = SLAFArray("data.slaf")
        >>> adata = LazyAnnData(slaf_array)
        >>> umap = adata.obsm["X_umap"]
        >>> print(f"UMAP shape: {umap.shape}")
        UMAP shape: (1000, 2)

        >>> # Create a new embedding
        >>> adata.obsm["X_pca"] = pca_coords  # shape: (1000, 50)
        >>> assert "X_pca" in adata.obsm
    """

    def __init__(self, lazy_adata: "LazyAnnData"):
        """
        Initialize LazyObsmView with LazyAnnData.

        Args:
            lazy_adata: LazyAnnData instance containing the single-cell data.
        """
        super().__init__()
        self.lazy_adata = lazy_adata
        self._slaf_array = lazy_adata.slaf
        # Required by LazySparseMixin
        self.slaf_array = lazy_adata.slaf
        self._shape = lazy_adata.shape  # (n_cells, n_genes)
        self.table_name = "cells"
        self.id_column = "cell_integer_id"
        self.table_type = "obsm"

    @property
    def shape(self) -> tuple[int, int]:
        """Required by LazySparseMixin - uses parent's shape"""
        return self._shape

Attributes¶

`shape: tuple[int, int]` `property` ¶

Required by LazySparseMixin - uses parent's shape

Functions¶

`init(lazy_adata: LazyAnnData)` ¶

Initialize LazyObsmView with LazyAnnData.

Parameters:

Name	Type	Description	Default
`lazy_adata`	`LazyAnnData`	LazyAnnData instance containing the single-cell data.	required

Source code in slaf/integrations/anndata.py

def __init__(self, lazy_adata: "LazyAnnData"):
    """
    Initialize LazyObsmView with LazyAnnData.

    Args:
        lazy_adata: LazyAnnData instance containing the single-cell data.
    """
    super().__init__()
    self.lazy_adata = lazy_adata
    self._slaf_array = lazy_adata.slaf
    # Required by LazySparseMixin
    self.slaf_array = lazy_adata.slaf
    self._shape = lazy_adata.shape  # (n_cells, n_genes)
    self.table_name = "cells"
    self.id_column = "cell_integer_id"
    self.table_type = "obsm"

`LazyVarmView` ¶

Bases: LazyMetadataViewMixin

Dictionary-like view of varm (multi-dimensional var annotations).

LazyVarmView provides a dictionary-like interface for accessing and mutating multi-dimensional gene annotations (e.g., PCA loadings) stored as separate columns in the genes.lance table. Each key maps to a 2D numpy array.

Key Features

Dictionary-like interface: varm["PCs"], "PCs" in varm, len(varm)
Multi-dimensional arrays: stored as FixedSizeListArray columns (native Lance vector type)
Schema-based detection: automatically detects vector columns from Lance schema
Selector support: respects gene selectors from parent LazyAnnData
Immutability: prevents deletion/modification of converted embeddings
Config.json consistency: reads from config for fast key discovery

Examples:

>>> # Access gene loadings
>>> slaf_array = SLAFArray("data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> pcs = adata.varm["PCs"]
>>> print(f"PCs shape: {pcs.shape}")
PCs shape: (20000, 50)

Source code in slaf/integrations/anndata.py

class LazyVarmView(LazyMetadataViewMixin):
    """
    Dictionary-like view of varm (multi-dimensional var annotations).

    LazyVarmView provides a dictionary-like interface for accessing and mutating
    multi-dimensional gene annotations (e.g., PCA loadings) stored as separate
    columns in the genes.lance table. Each key maps to a 2D numpy array.

    Key Features:
        - Dictionary-like interface: varm["PCs"], "PCs" in varm, len(varm)
        - Multi-dimensional arrays: stored as FixedSizeListArray columns (native Lance vector type)
        - Schema-based detection: automatically detects vector columns from Lance schema
        - Selector support: respects gene selectors from parent LazyAnnData
        - Immutability: prevents deletion/modification of converted embeddings
        - Config.json consistency: reads from config for fast key discovery

    Examples:
        >>> # Access gene loadings
        >>> slaf_array = SLAFArray("data.slaf")
        >>> adata = LazyAnnData(slaf_array)
        >>> pcs = adata.varm["PCs"]
        >>> print(f"PCs shape: {pcs.shape}")
        PCs shape: (20000, 50)
    """

    def __init__(self, lazy_adata: "LazyAnnData"):
        """
        Initialize LazyVarmView with LazyAnnData.

        Args:
            lazy_adata: LazyAnnData instance containing the single-cell data.
        """
        super().__init__()
        self.lazy_adata = lazy_adata
        self._slaf_array = lazy_adata.slaf
        # Required by LazySparseMixin
        self.slaf_array = lazy_adata.slaf
        self._shape = lazy_adata.shape  # (n_cells, n_genes)
        self.table_name = "genes"
        self.id_column = "gene_integer_id"
        self.table_type = "varm"

    @property
    def shape(self) -> tuple[int, int]:
        """Required by LazySparseMixin - uses parent's shape"""
        return self._shape

Attributes¶

`shape: tuple[int, int]` `property` ¶

Required by LazySparseMixin - uses parent's shape

Functions¶

`init(lazy_adata: LazyAnnData)` ¶

Initialize LazyVarmView with LazyAnnData.

Parameters:

Name	Type	Description	Default
`lazy_adata`	`LazyAnnData`	LazyAnnData instance containing the single-cell data.	required

Source code in slaf/integrations/anndata.py

def __init__(self, lazy_adata: "LazyAnnData"):
    """
    Initialize LazyVarmView with LazyAnnData.

    Args:
        lazy_adata: LazyAnnData instance containing the single-cell data.
    """
    super().__init__()
    self.lazy_adata = lazy_adata
    self._slaf_array = lazy_adata.slaf
    # Required by LazySparseMixin
    self.slaf_array = lazy_adata.slaf
    self._shape = lazy_adata.shape  # (n_cells, n_genes)
    self.table_name = "genes"
    self.id_column = "gene_integer_id"
    self.table_type = "varm"

`LazyUnsView` ¶

Bases: LazyDictionaryViewMixin

Dictionary-like view of uns (unstructured metadata).

LazyUnsView provides a dictionary-like interface for accessing and mutating unstructured metadata stored in uns.json. Unlike obs/var/obsm/varm, uns metadata is always mutable and stored as JSON.

Key Features

Dictionary-like interface: uns["key"], "key" in uns, len(uns)
JSON storage: stored in uns.json file
Always mutable: no immutability tracking
JSON serialization: automatically converts numpy/pandas objects

Examples:

>>> # Store metadata
>>> slaf_array = SLAFArray("data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> adata.uns["neighbors"] = {"params": {"n_neighbors": 15}}
>>> print(adata.uns["neighbors"])
{'params': {'n_neighbors': 15}}

Source code in slaf/integrations/anndata.py

class LazyUnsView(LazyDictionaryViewMixin):
    """
    Dictionary-like view of uns (unstructured metadata).

    LazyUnsView provides a dictionary-like interface for accessing and mutating
    unstructured metadata stored in uns.json. Unlike obs/var/obsm/varm, uns
    metadata is always mutable and stored as JSON.

    Key Features:
        - Dictionary-like interface: uns["key"], "key" in uns, len(uns)
        - JSON storage: stored in uns.json file
        - Always mutable: no immutability tracking
        - JSON serialization: automatically converts numpy/pandas objects

    Examples:
        >>> # Store metadata
        >>> slaf_array = SLAFArray("data.slaf")
        >>> adata = LazyAnnData(slaf_array)
        >>> adata.uns["neighbors"] = {"params": {"n_neighbors": 15}}
        >>> print(adata.uns["neighbors"])
        {'params': {'n_neighbors': 15}}
    """

    def __init__(self, lazy_adata: "LazyAnnData"):
        """
        Initialize LazyUnsView with LazyAnnData.

        Args:
            lazy_adata: LazyAnnData instance containing the single-cell data.
        """
        self.lazy_adata = lazy_adata
        self._slaf_array = lazy_adata.slaf
        self._uns_path = self._slaf_array._join_path(
            self._slaf_array.slaf_path, "uns.json"
        )
        self._uns_data: dict | None = None

    def _load_uns(self) -> dict:
        """Load uns.json if it exists, otherwise return empty dict"""
        if self._uns_data is not None:
            return self._uns_data

        # Check if file exists (not directory)
        import os

        if self._slaf_array._is_cloud_path(self._uns_path):
            # For cloud paths, try to open the file
            try:
                with self._slaf_array._open_file(self._uns_path) as f:
                    import json

                    self._uns_data = json.load(f)
            except Exception:
                self._uns_data = {}
        else:
            # For local paths, check if file exists
            if os.path.exists(self._uns_path) and os.path.isfile(self._uns_path):
                with self._slaf_array._open_file(self._uns_path) as f:
                    import json

                    self._uns_data = json.load(f)
            else:
                self._uns_data = {}

        return self._uns_data

    def keys(self) -> list[str]:
        """List all available keys"""
        return list(self._load_uns().keys())

    def __getitem__(self, key: str) -> Any:
        """Retrieve metadata value"""
        uns_data = self._load_uns()
        if key not in uns_data:
            raise KeyError(f"uns key '{key}' not found")
        return uns_data[key]

    def __setitem__(self, key: str, value: Any):
        """Store metadata value"""
        uns_data = self._load_uns()

        # Validate key name
        self._validate_name(key)

        # Convert numpy arrays and pandas objects to JSON-serializable
        value = self._json_serialize(value)

        uns_data[key] = value
        self._save_uns(uns_data)

    def __delitem__(self, key: str):
        """Delete metadata key"""
        uns_data = self._load_uns()
        if key not in uns_data:
            raise KeyError(f"uns key '{key}' not found")
        del uns_data[key]
        self._save_uns(uns_data)

    def _json_serialize(self, value: Any) -> Any:
        """Convert value to JSON-serializable format"""
        import pandas as pd

        if isinstance(value, np.ndarray):
            return value.tolist()
        elif isinstance(value, np.integer | np.floating):
            return value.item()
        elif isinstance(value, pd.Series):
            return value.tolist()
        elif isinstance(value, dict):
            return {k: self._json_serialize(v) for k, v in value.items()}
        elif isinstance(value, list):
            return [self._json_serialize(item) for item in value]
        return value

    def _save_uns(self, uns_data: dict):
        """Save uns.json atomically"""
        import json

        # Use same cloud-compatible file writing as config.json
        with self._slaf_array._open_file(self._uns_path, "w") as f:
            json.dump(uns_data, f, indent=2)

        # Invalidate cache
        self._uns_data = uns_data

Functions¶

`init(lazy_adata: LazyAnnData)` ¶

Initialize LazyUnsView with LazyAnnData.

Parameters:

Name	Type	Description	Default
`lazy_adata`	`LazyAnnData`	LazyAnnData instance containing the single-cell data.	required

Source code in slaf/integrations/anndata.py

def __init__(self, lazy_adata: "LazyAnnData"):
    """
    Initialize LazyUnsView with LazyAnnData.

    Args:
        lazy_adata: LazyAnnData instance containing the single-cell data.
    """
    self.lazy_adata = lazy_adata
    self._slaf_array = lazy_adata.slaf
    self._uns_path = self._slaf_array._join_path(
        self._slaf_array.slaf_path, "uns.json"
    )
    self._uns_data: dict | None = None

`keys() -> list[str]` ¶

List all available keys

Source code in slaf/integrations/anndata.py

def keys(self) -> list[str]:
    """List all available keys"""
    return list(self._load_uns().keys())

`LazyAnnData` ¶

Bases: LazySparseMixin

AnnData-compatible interface for SLAF data with lazy evaluation.

LazyAnnData provides a drop-in replacement for AnnData objects that works with SLAF datasets. It implements lazy evaluation to avoid loading all data into memory, making it suitable for large single-cell datasets.

Key Features

AnnData-compatible interface
Lazy evaluation for memory efficiency
Support for cell and gene subsetting
Integration with scanpy workflows
Automatic metadata loading
Transformation caching

Examples:

>>> # Basic usage
>>> slaf_array = SLAFArray("path/to/data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> print(f"AnnData shape: {adata.shape}")
AnnData shape: (1000, 20000)

>>> # Access expression data
>>> print(f"Expression matrix shape: {adata.X.shape}")
Expression matrix shape: (1000, 20000)

>>> # Access metadata
>>> print(f"Cell metadata columns: {list(adata.obs.columns)}")
Cell metadata columns: ['cell_type', 'total_counts', 'batch']
>>> print(f"Gene metadata columns: {list(adata.var.columns)}")
Gene metadata columns: ['gene_type', 'chromosome']

>>> # Subsetting operations
>>> subset = adata[:100, :5000]  # First 100 cells, first 5000 genes
>>> print(f"Subset shape: {subset.shape}")
Subset shape: (100, 5000)

Source code in slaf/integrations/anndata.py

class LazyAnnData(LazySparseMixin):
    """
    AnnData-compatible interface for SLAF data with lazy evaluation.

    LazyAnnData provides a drop-in replacement for AnnData objects that works with
    SLAF datasets. It implements lazy evaluation to avoid loading all data into memory,
    making it suitable for large single-cell datasets.

    Key Features:
        - AnnData-compatible interface
        - Lazy evaluation for memory efficiency
        - Support for cell and gene subsetting
        - Integration with scanpy workflows
        - Automatic metadata loading
        - Transformation caching

    Examples:
        >>> # Basic usage
        >>> slaf_array = SLAFArray("path/to/data.slaf")
        >>> adata = LazyAnnData(slaf_array)
        >>> print(f"AnnData shape: {adata.shape}")
        AnnData shape: (1000, 20000)

        >>> # Access expression data
        >>> print(f"Expression matrix shape: {adata.X.shape}")
        Expression matrix shape: (1000, 20000)

        >>> # Access metadata
        >>> print(f"Cell metadata columns: {list(adata.obs.columns)}")
        Cell metadata columns: ['cell_type', 'total_counts', 'batch']
        >>> print(f"Gene metadata columns: {list(adata.var.columns)}")
        Gene metadata columns: ['gene_type', 'chromosome']

        >>> # Subsetting operations
        >>> subset = adata[:100, :5000]  # First 100 cells, first 5000 genes
        >>> print(f"Subset shape: {subset.shape}")
        Subset shape: (100, 5000)
    """

    def __init__(
        self,
        slaf_array: SLAFArray,
        backend: str = "auto",
    ):
        """
        Initialize LazyAnnData with SLAF array.

        Args:
            slaf_array: SLAFArray instance containing the single-cell data.
                       Used for database queries and metadata access.
            backend: Backend for expression matrix. Currently supports "scipy" and "auto".
                    "auto" defaults to "scipy" for sparse matrix operations.

        Raises:
            ValueError: If the backend is not supported.
            RuntimeError: If the SLAF array is not properly initialized.

        Examples:
            >>> # Basic initialization
            >>> slaf_array = SLAFArray("path/to/data.slaf")
            >>> adata = LazyAnnData(slaf_array)
            >>> print(f"Backend: {adata.backend}")
            Backend: auto

            >>> # With explicit backend
            >>> adata = LazyAnnData(slaf_array, backend="scipy")
            >>> print(f"Backend: {adata.backend}")
            Backend: scipy

            >>> # Error handling for unsupported backend
            >>> try:
            ...     adata = LazyAnnData(slaf_array, backend="unsupported")
            ... except ValueError as e:
            ...     print(f"Error: {e}")
            Error: Unknown backend: unsupported
        """
        super().__init__()
        self.slaf = slaf_array
        self.backend = backend

        # Initialize expression matrix
        if backend == "scipy" or backend == "auto":
            self._X = LazyExpressionMatrix(slaf_array)
            self._X.parent_adata = self  # Set parent reference for transformations
        else:
            raise ValueError(f"Unknown backend: {backend}")

        # Lazy-loaded metadata
        self._obs = None
        self._var = None
        self._cached_obs_names: pd.Index | None = None
        self._cached_var_names: pd.Index | None = None

        # Filter selectors for subsetting
        self._cell_selector: Any = None
        self._gene_selector: Any = None
        self._filtered_obs: Callable[[], pd.DataFrame] | None = None
        self._filtered_var: Callable[[], pd.DataFrame] | None = None

        # Reference to parent LazyAnnData (for sliced objects)
        self._parent_adata: LazyAnnData | None = None

        # Transformations for lazy evaluation
        self._transformations: dict[str, Any] = {}

        # Layers view (lazy initialization)
        self._layers: LazyLayersView | None = None

    @property
    def layers(self) -> LazyLayersView:
        """
        Access to layers (dictionary-like interface).

        Returns a dictionary-like view of layers that provides access to alternative
        representations of the expression matrix (e.g., spliced, unspliced, counts).
        Layers have the same dimensions as X.

        Returns:
            LazyLayersView providing dictionary-like access to layers.

        Examples:
            >>> # Access a layer
            >>> slaf_array = SLAFArray("data.slaf")
            >>> adata = LazyAnnData(slaf_array)
            >>> spliced = adata.layers["spliced"]
            >>> print(f"Layer shape: {spliced.shape}")
            Layer shape: (1000, 20000)

            >>> # List available layers
            >>> print(list(adata.layers.keys()))
            ['spliced', 'unspliced']

            >>> # Check if layer exists
            >>> assert "spliced" in adata.layers
        """
        if self._layers is None:
            self._layers = LazyLayersView(self)
        return self._layers

    @property
    def obs(self) -> LazyObsView:
        """
        Cell metadata (observations) - mutable view with DataFrame-like and dict-like interfaces.

        Returns a view that provides both DataFrame-like and dictionary-like access to cell
        metadata columns with support for creating, updating, and deleting columns. This view
        respects cell selectors from parent LazyAnnData.

        When accessed with a string key (e.g., ``obs["cluster"]``), returns a pandas Series
        (AnnData-compatible). When accessed directly, behaves like a DataFrame.

        Returns:
            LazyObsView providing DataFrame-like and dictionary-like access to obs columns.

        Examples:
            >>> # DataFrame-like access (AnnData-compatible)
            >>> slaf_array = SLAFArray("data.slaf")
            >>> adata = LazyAnnData(slaf_array)
            >>> df = adata.obs  # Returns DataFrame-like view
            >>> cluster = adata.obs["cluster"]  # Returns pd.Series (AnnData-compatible)
            >>> print(cluster.head())

            >>> # Create a new column
            >>> adata.obs["new_cluster"] = new_cluster_labels
            >>> assert "new_cluster" in adata.obs

            >>> # List available columns
            >>> print(list(adata.obs.keys()))
            ['cell_id', 'total_counts', 'cluster', 'new_cluster']
        """
        if not hasattr(self, "_obs_view"):
            self._obs_view = LazyObsView(self)
        return self._obs_view

    @property
    def var(self) -> LazyVarView:
        """
        Gene metadata (variables) - mutable view with DataFrame-like and dict-like interfaces.

        Returns a view that provides both DataFrame-like and dictionary-like access to gene
        metadata columns with support for creating, updating, and deleting columns. This view
        respects gene selectors from parent LazyAnnData.

        When accessed with a string key (e.g., ``var["highly_variable"]``), returns a pandas
        Series (AnnData-compatible). When accessed directly, behaves like a DataFrame.

        Returns:
            LazyVarView providing DataFrame-like and dictionary-like access to var columns.

        Examples:
            >>> # DataFrame-like access (AnnData-compatible)
            >>> slaf_array = SLAFArray("data.slaf")
            >>> adata = LazyAnnData(slaf_array)
            >>> df = adata.var  # Returns DataFrame-like view
            >>> hvg = adata.var["highly_variable"]  # Returns pd.Series (AnnData-compatible)
            >>> print(hvg.head())

            >>> # Create a new column
            >>> adata.var["new_annotation"] = new_annotations
            >>> assert "new_annotation" in adata.var
        """
        if not hasattr(self, "_var_view"):
            self._var_view = LazyVarView(self)
        return self._var_view

    @property
    def obsm(self) -> LazyObsmView:
        """
        Multi-dimensional obs annotations (embeddings, PCA, etc.).

        Returns a dictionary-like view that provides access to multi-dimensional
        cell annotations stored as separate columns in cells.lance. Each key maps
        to a 2D numpy array.

        Returns:
            LazyObsmView providing dictionary-like access to obsm keys.

        Examples:
            >>> # Access an embedding
            >>> slaf_array = SLAFArray("data.slaf")
            >>> adata = LazyAnnData(slaf_array)
            >>> umap = adata.obsm["X_umap"]
            >>> print(f"UMAP shape: {umap.shape}")
            UMAP shape: (1000, 2)

            >>> # Create a new embedding
            >>> adata.obsm["X_pca"] = pca_coords  # shape: (1000, 50)
            >>> assert "X_pca" in adata.obsm
        """
        if not hasattr(self, "_obsm"):
            self._obsm = LazyObsmView(self)
        return self._obsm

    @property
    def varm(self) -> LazyVarmView:
        """
        Multi-dimensional var annotations (gene loadings, etc.).

        Returns a dictionary-like view that provides access to multi-dimensional
        gene annotations stored as separate columns in genes.lance. Each key maps
        to a 2D numpy array.

        Returns:
            LazyVarmView providing dictionary-like access to varm keys.

        Examples:
            >>> # Access gene loadings
            >>> slaf_array = SLAFArray("data.slaf")
            >>> adata = LazyAnnData(slaf_array)
            >>> pcs = adata.varm["PCs"]
            >>> print(f"PCs shape: {pcs.shape}")
            PCs shape: (20000, 50)
        """
        if not hasattr(self, "_varm"):
            self._varm = LazyVarmView(self)
        return self._varm

    @property
    def uns(self) -> LazyUnsView:
        """
        Unstructured metadata (analysis parameters, etc.).

        Returns a dictionary-like view that provides access to unstructured
        metadata stored in uns.json. Unlike obs/var/obsm/varm, uns metadata
        is always mutable and stored as JSON.

        Returns:
            LazyUnsView providing dictionary-like access to uns keys.

        Examples:
            >>> # Store metadata
            >>> slaf_array = SLAFArray("data.slaf")
            >>> adata = LazyAnnData(slaf_array)
            >>> adata.uns["neighbors"] = {"params": {"n_neighbors": 15}}
            >>> print(adata.uns["neighbors"])
            {'params': {'n_neighbors': 15}}
        """
        if not hasattr(self, "_uns"):
            self._uns = LazyUnsView(self)
        return self._uns

    @property
    def X(self) -> LazyExpressionMatrix:
        """
        Access to expression data.

        Returns the lazy expression matrix that provides scipy.sparse-compatible
        access to the single-cell expression data. The matrix is lazily evaluated
        to avoid loading all data into memory.

        Returns:
            LazyExpressionMatrix providing scipy.sparse-compatible interface.

        Examples:
            >>> # Access expression data
            >>> slaf_array = SLAFArray("path/to/data.slaf")
            >>> adata = LazyAnnData(slaf_array)
            >>> print(f"Expression matrix shape: {adata.X.shape}")
            Expression matrix shape: (1000, 20000)

            >>> # Subsetting expression data
            >>> subset_X = adata.X[:100, :5000]
            >>> print(f"Subset expression shape: {subset_X.shape}")
            Subset expression shape: (100, 5000)

            >>> # Check matrix type
            >>> print(f"Matrix type: {type(adata.X)}")
            Matrix type: <class 'slaf.integrations.anndata.LazyExpressionMatrix'>
        """
        return self._X

    @property
    def obs_deprecated(self) -> pd.DataFrame:
        """
        Cell metadata (observations) - DEPRECATED.

        .. deprecated:: 0.X
            This property is deprecated. Use :attr:`obs` (formerly :attr:`obs`) instead.
            This will be removed in a future version.

        This property triggers computation of metadata when accessed.
        For lazy access to metadata structure only, use obs.columns, obs.index, etc.
        """
        import warnings

        warnings.warn(
            "obs_deprecated is deprecated and will be removed in a future version. "
            "Use obs instead, which provides both DataFrame-like and dict-like access "
            "with mutation support.",
            DeprecationWarning,
            stacklevel=2,
        )
        if self._filtered_obs is not None:
            result = self._filtered_obs()
            if isinstance(result, pd.DataFrame):
                return result
            return pd.DataFrame()
        if self._obs is None:
            # Use the obs from SLAFArray (now polars DataFrame)
            obs_df = getattr(self.slaf, "obs", None)
            if obs_df is not None:
                # Work with polars DataFrame internally
                obs_pl = obs_df
                # Drop cell_integer_id column if present to match AnnData expectations
                if "cell_integer_id" in obs_pl.columns:
                    obs_pl = obs_pl.drop("cell_integer_id")
                # Filter out vector columns (obsm) - these are FixedSizeListArray columns
                # Vector columns should only be accessed via adata.obsm, not adata.obs
                cells_table = getattr(self.slaf, "cells", None)
                if cells_table is not None:
                    schema = cells_table.schema
                    vector_column_names = {
                        field.name
                        for field in schema
                        if isinstance(field.type, pa.FixedSizeListType)
                    }
                    # Drop vector columns from obs
                    columns_to_drop = [
                        col for col in obs_pl.columns if col in vector_column_names
                    ]
                    if columns_to_drop:
                        obs_pl = obs_pl.drop(columns_to_drop)
                # Convert to pandas DataFrame for AnnData compatibility at API boundary
                obs_copy = obs_pl.to_pandas()
                # Set cell_id as index if present, otherwise use default index
                if "cell_id" in obs_copy.columns:
                    obs_copy = obs_copy.set_index("cell_id")
                # Set index name to match AnnData format
                if hasattr(obs_copy, "index"):
                    obs_copy.index.name = "cell_id"
                self._obs = obs_copy
            else:
                self._obs = pd.DataFrame()
        return self._obs

    @property
    def var_deprecated(self) -> pd.DataFrame:
        """
        Gene metadata (variables) - DEPRECATED.

        .. deprecated:: 0.X
            This property is deprecated. Use :attr:`var` (formerly :attr:`var`) instead.
            This will be removed in a future version.

        This property triggers computation of metadata when accessed.
        For lazy access to metadata structure only, use var.columns, var.index, etc.
        """
        import warnings

        warnings.warn(
            "var_deprecated is deprecated and will be removed in a future version. "
            "Use var instead, which provides both DataFrame-like and dict-like access "
            "with mutation support.",
            DeprecationWarning,
            stacklevel=2,
        )
        if self._filtered_var is not None:
            result = self._filtered_var()
            if isinstance(result, pd.DataFrame):
                return result
            return pd.DataFrame()
        if self._var is None:
            var_df = getattr(self.slaf, "var", None)
            if var_df is not None:
                # Work with polars DataFrame internally
                var_pl = var_df
                # Drop gene_integer_id column if present to match AnnData expectations
                if "gene_integer_id" in var_pl.columns:
                    var_pl = var_pl.drop("gene_integer_id")
                # Filter out vector columns (varm) - these are FixedSizeListArray columns
                # Vector columns should only be accessed via adata.varm, not adata.var
                genes_table = getattr(self.slaf, "genes", None)
                if genes_table is not None:
                    schema = genes_table.schema
                    vector_column_names = {
                        field.name
                        for field in schema
                        if isinstance(field.type, pa.FixedSizeListType)
                    }
                    # Drop vector columns from var
                    columns_to_drop = [
                        col for col in var_pl.columns if col in vector_column_names
                    ]
                    if columns_to_drop:
                        var_pl = var_pl.drop(columns_to_drop)
                # Convert to pandas DataFrame for AnnData compatibility at API boundary
                var_copy = var_pl.to_pandas()
                # Set gene_id as index if present, otherwise use default index
                if "gene_id" in var_copy.columns:
                    var_copy = var_copy.set_index("gene_id")
                # Set index name to match AnnData format
                if hasattr(var_copy, "index"):
                    var_copy.index.name = "gene_id"
                self._var = var_copy
            else:
                self._var = pd.DataFrame()
        return self._var

    @property
    def obs_names(self) -> pd.Index:
        """Cell names"""
        if self._cached_obs_names is None:
            # Use the DataFrame's index from the view
            self._cached_obs_names = self.obs._get_dataframe().index
        return self._cached_obs_names

    @property
    def var_names(self) -> pd.Index:
        """Gene names"""
        if self._cached_var_names is None:
            # Use the DataFrame's index from the view
            self._cached_var_names = self.var._get_dataframe().index
        return self._cached_var_names

    @property
    def n_obs(self) -> int:
        """Number of observations (cells)"""
        return self.shape[0]

    @property
    def n_vars(self) -> int:
        """Number of variables (genes)"""
        return self.shape[1]

    @property
    def shape(self) -> tuple[int, int]:
        """Get the shape of the data, accounting for any applied filters"""
        if (
            getattr(self, "_cell_selector", None) is not None
            or getattr(self, "_gene_selector", None) is not None
        ):
            # Calculate the shape based on selectors
            cell_selector = (
                self._cell_selector
                if getattr(self, "_cell_selector", None) is not None
                else slice(None)
            )
            gene_selector = (
                self._gene_selector
                if getattr(self, "_gene_selector", None) is not None
                else slice(None)
            )

            # Use the same logic as _get_result_shape in LazySparseMixin
            n_cells = self._calculate_selected_count(cell_selector, axis=0)
            n_genes = self._calculate_selected_count(gene_selector, axis=1)

            return (n_cells, n_genes)
        else:
            # No filters applied, return original shape
            return self.slaf.shape

    def _calculate_selected_count(self, selector, axis: int) -> int:
        """Calculate the number of selected entities for a given selector"""
        if selector is None or (
            isinstance(selector, slice) and selector == slice(None)
        ):
            return self.slaf.shape[axis]

        if isinstance(selector, slice):
            start = selector.start or 0
            stop = selector.stop or self.slaf.shape[axis]
            step = selector.step or 1

            # Handle negative indices
            if start < 0:
                start = self.slaf.shape[axis] + start
            if stop < 0:
                stop = self.slaf.shape[axis] + stop

            # Clamp bounds to actual data size
            start = max(0, min(start, self.slaf.shape[axis]))
            stop = max(0, min(stop, self.slaf.shape[axis]))

            return len(range(start, stop, step))
        elif isinstance(selector, list | np.ndarray):
            if isinstance(selector, np.ndarray) and selector.dtype == bool:
                return np.sum(selector)
            return len(selector)
        elif isinstance(selector, int | np.integer):
            return 1
        else:
            return self.slaf.shape[axis]

    def __getitem__(self, key) -> "LazyAnnData":
        """Subset the data, composing selectors if already sliced"""
        # Disallow chained slicing on LazyAnnData objects
        if self._cell_selector is not None or self._gene_selector is not None:
            raise NotImplementedError(
                "Chained slicing on LazyAnnData objects is not supported. "
                "Please use adata[rows, cols] for single-step slicing instead of "
                "adata[rows][:, cols]. For chained slicing on the expression matrix, "
                "use adata.X[rows][:, cols]."
            )

        # Parse indexing key using the same logic as LazyExpressionMatrix
        cell_selector, gene_selector = self._parse_key(key)

        # Create a new LazyAnnData with the same backend
        new_adata = LazyAnnData(self.slaf, backend=self.backend)

        # Store reference to parent for accessing full DataFrames
        new_adata._parent_adata = self

        # Store the selectors for lazy filtering
        new_adata._cell_selector = cell_selector
        new_adata._gene_selector = gene_selector

        # Update the LazyExpressionMatrix to know about the slicing
        if isinstance(new_adata._X, LazyExpressionMatrix):
            new_adata._X.parent_adata = new_adata
            # Store the selectors in the LazyExpressionMatrix so it can use them
            new_adata._X._cell_selector = cell_selector
            new_adata._X._gene_selector = gene_selector
            # Update the shape to reflect the slicing
            new_adata._X._update_shape()

        # Override obs and var properties to apply filtering
        def filtered_obs() -> pd.DataFrame:
            # Always apply the composed selector to the original obs DataFrame
            # Get the FULL DataFrame from parent (without selectors) to apply our selector
            # Access parent's view without selectors
            parent = getattr(self, "_parent_adata", None)
            if parent is None:
                parent = self
            parent_obs_view = parent.obs
            # Temporarily clear selectors to get full DataFrame
            original_cell_selector = getattr(parent, "_cell_selector", None)
            original_gene_selector = getattr(parent, "_gene_selector", None)
            # Temporarily clear selectors
            parent._cell_selector = None
            parent._gene_selector = None
            # Clear cache to force reload
            if hasattr(parent_obs_view, "_dataframe"):
                parent_obs_view._dataframe = None
            # Get full DataFrame
            obs_df = parent_obs_view._get_dataframe()
            # Restore selectors
            parent._cell_selector = original_cell_selector
            parent._gene_selector = original_gene_selector
            if cell_selector is None or (
                isinstance(cell_selector, slice) and cell_selector == slice(None)
            ):
                pass
            else:
                obs_df = obs_df.copy()
                if isinstance(cell_selector, slice):
                    start = cell_selector.start
                    stop = cell_selector.stop
                    step = cell_selector.step if cell_selector.step is not None else 1

                    # Handle None values
                    if start is None:
                        start = 0
                    if stop is None:
                        stop = len(obs_df)

                    # Handle negative indices
                    if start < 0:
                        start = len(obs_df) + start
                    if stop < 0:
                        stop = len(obs_df) + stop

                    # Clamp bounds to valid range
                    start = max(0, min(start, len(obs_df)))
                    stop = max(0, min(stop, len(obs_df)))
                    obs_df = obs_df.iloc[start:stop:step]
                elif (
                    isinstance(cell_selector, np.ndarray)
                    and cell_selector.dtype == bool
                ):
                    # Boolean mask - ensure it matches the length
                    if len(cell_selector) == len(obs_df):
                        obs_df = obs_df[cell_selector]
                    else:
                        # Pad or truncate the mask to match
                        if len(cell_selector) < len(obs_df):
                            mask: np.ndarray = np.zeros(len(obs_df), dtype=bool)
                            mask[: len(cell_selector)] = cell_selector
                        else:
                            mask = cell_selector[: len(obs_df)]
                        obs_df = obs_df[mask]
                elif isinstance(cell_selector, list | np.ndarray):
                    # Integer indices
                    obs_df = obs_df.iloc[cell_selector]
                elif isinstance(cell_selector, int | np.integer):
                    obs_df = obs_df.iloc[[int(cell_selector)]]
                # else: leave as is
            # Remove unused categories for all categorical columns if not empty
            if obs_df is not None and not obs_df.empty:
                for col in obs_df.select_dtypes(include="category").columns:
                    col_data = obs_df[col]
                    # Type-safe check for pandas categorical data
                    if isinstance(col_data, pd.Series) and hasattr(col_data, "cat"):
                        obs_df[col] = col_data.cat.remove_unused_categories()
            if isinstance(obs_df, pd.DataFrame):
                return obs_df
            return pd.DataFrame()

        def filtered_var() -> pd.DataFrame:
            # Always apply the composed selector to the original var DataFrame
            # Get the FULL DataFrame from parent (without selectors) to apply our selector
            # Access parent's view without selectors
            parent = getattr(self, "_parent_adata", None)
            if parent is None:
                parent = self
            parent_var_view = parent.var
            # Temporarily clear selectors to get full DataFrame
            original_cell_selector = getattr(parent, "_cell_selector", None)
            original_gene_selector = getattr(parent, "_gene_selector", None)
            # Temporarily clear selectors
            parent._cell_selector = None
            parent._gene_selector = None
            # Clear cache to force reload
            if hasattr(parent_var_view, "_dataframe"):
                parent_var_view._dataframe = None
            # Get full DataFrame
            var_df = parent_var_view._get_dataframe()
            # Restore selectors
            parent._cell_selector = original_cell_selector
            parent._gene_selector = original_gene_selector

            if gene_selector is None or (
                isinstance(gene_selector, slice) and gene_selector == slice(None)
            ):
                pass
            else:
                var_df = var_df.copy()
                if isinstance(gene_selector, slice):
                    start = gene_selector.start
                    stop = gene_selector.stop
                    step = gene_selector.step if gene_selector.step is not None else 1

                    # Handle None values
                    if start is None:
                        start = 0
                    if stop is None:
                        stop = len(var_df)

                    # Handle negative indices
                    if start < 0:
                        start = len(var_df) + start
                    if stop < 0:
                        stop = len(var_df) + stop

                    # Clamp bounds to valid range
                    start = max(0, min(start, len(var_df)))
                    stop = max(0, min(stop, len(var_df)))
                    var_df = var_df.iloc[start:stop:step]
                elif (
                    isinstance(gene_selector, np.ndarray)
                    and gene_selector.dtype == bool
                ):
                    # Boolean mask - ensure it matches the length
                    if len(gene_selector) == len(var_df):
                        var_df = var_df[gene_selector]
                    else:
                        # Pad or truncate the mask to match
                        if len(gene_selector) < len(var_df):
                            mask: np.ndarray = np.zeros(len(var_df), dtype=bool)
                            mask[: len(gene_selector)] = gene_selector
                        else:
                            mask = gene_selector[: len(var_df)]
                        var_df = var_df[mask]
                elif isinstance(gene_selector, list | np.ndarray):
                    # Integer indices
                    var_df = var_df.iloc[gene_selector]
                elif isinstance(gene_selector, int | np.integer):
                    var_df = var_df.iloc[[int(gene_selector)]]
                # else: leave as is
            # Remove unused categories for all categorical columns if not empty
            if var_df is not None and not var_df.empty:
                for col in var_df.select_dtypes(include="category").columns:
                    col_data = var_df[col]
                    # Type-safe check for pandas categorical data
                    if isinstance(col_data, pd.Series) and hasattr(col_data, "cat"):
                        var_df[col] = col_data.cat.remove_unused_categories()
            if isinstance(var_df, pd.DataFrame):
                return var_df
            return pd.DataFrame()

        # Store the filter functions
        new_adata._filtered_obs = filtered_obs
        new_adata._filtered_var = filtered_var

        # Ensure obs_names and var_names match the filtered DataFrames
        new_adata._cached_obs_names = new_adata._filtered_obs().index
        new_adata._cached_var_names = new_adata._filtered_var().index

        # Copy transformations
        new_adata._transformations = self._transformations.copy()

        # Copy layers view (will be recreated with new selectors when accessed)
        new_adata._layers = None

        return new_adata

    def copy(self) -> "LazyAnnData":
        """Create a copy"""
        new_adata = LazyAnnData(self.slaf, backend=self.backend)

        # Copy selectors
        new_adata._cell_selector = self._cell_selector
        new_adata._gene_selector = self._gene_selector

        # Copy filtered metadata functions
        new_adata._filtered_obs = self._filtered_obs
        new_adata._filtered_var = self._filtered_var

        # Copy transformations
        new_adata._transformations = self._transformations.copy()

        # Set up parent reference for the expression matrix
        if isinstance(new_adata._X, LazyExpressionMatrix):
            new_adata._X.parent_adata = new_adata

        return new_adata

    def to_memory(self) -> scipy.sparse.spmatrix:
        """Load entire matrix into memory"""
        return self.get_expression_data()

    def write(self, filename: str):
        """Write to h5ad format (would need implementation)"""
        raise NotImplementedError("Writing LazyAnnData not yet implemented")

    def get_expression_data(self) -> scipy.sparse.csr_matrix:
        """Get expression data with any applied filters"""
        if isinstance(self._X, LazyExpressionMatrix):
            # If the LazyExpressionMatrix already has selectors stored, just get the data
            # without passing additional selectors to avoid double-composition
            if self._X._cell_selector is not None or self._X._gene_selector is not None:
                return self._X[:, :].compute()
            elif self._cell_selector is not None or self._gene_selector is not None:
                cell_sel = (
                    self._cell_selector
                    if self._cell_selector is not None
                    else slice(None)
                )
                gene_sel = (
                    self._gene_selector
                    if self._gene_selector is not None
                    else slice(None)
                )
                return self._X[cell_sel, gene_sel].compute()
            else:
                return self._X[:, :].compute()
        else:
            raise NotImplementedError(
                "Dask backend not yet implemented for get_expression_data"
            )

    def compute(self) -> "sc.AnnData":
        """Explicitly compute and return a native AnnData object"""
        import scanpy as sc

        # Create native AnnData object (get DataFrames from views)
        adata = sc.AnnData(
            X=self._X.compute(),
            obs=self.obs._get_dataframe(),
            var=self.var._get_dataframe(),
        )

        return adata

    def _update_with_normalized_data(
        self, result_df: pl.DataFrame, target_sum: float, inplace: bool
    ) -> "LazyAnnData | None":
        """
        Update AnnData with normalized data from fragment processing.

        Args:
            result_df: Polars DataFrame with normalized values
            target_sum: Target sum used for normalization
            inplace: Whether to modify in place

        Returns:
            Updated LazyAnnData or None if inplace=True
        """
        if inplace:
            # Store normalization transformation
            if not hasattr(self, "_transformations"):
                self._transformations = {}

            self._transformations["normalize_total"] = {
                "type": "normalize_total",
                "target_sum": target_sum,
                "fragment_processed": True,
                "result_df": result_df,
            }

            print(
                f"Applied normalize_total with target_sum={target_sum} (fragment processing)"
            )
            return None
        else:
            # Create a copy with the transformation
            new_adata = self.copy()
            if not hasattr(new_adata, "_transformations"):
                new_adata._transformations = {}

            new_adata._transformations["normalize_total"] = {
                "type": "normalize_total",
                "target_sum": target_sum,
                "fragment_processed": True,
                "result_df": result_df,
            }

            return new_adata

    def _update_with_log1p_data(
        self, result_df: pl.DataFrame, inplace: bool
    ) -> "LazyAnnData | None":
        """
        Update AnnData with log1p data from fragment processing.

        Args:
            result_df: Polars DataFrame with log1p values
            inplace: Whether to modify in place

        Returns:
            Updated LazyAnnData or None if inplace=True
        """
        if inplace:
            # Store log1p transformation
            if not hasattr(self, "_transformations"):
                self._transformations = {}

            self._transformations["log1p"] = {
                "type": "log1p",
                "fragment_processed": True,
                "result_df": result_df,
            }

            print("Applied log1p transformation (fragment processing)")
            return None
        else:
            # Create a copy with the transformation
            new_adata = self.copy()
            if not hasattr(new_adata, "_transformations"):
                new_adata._transformations = {}

            new_adata._transformations["log1p"] = {
                "type": "log1p",
                "fragment_processed": True,
                "result_df": result_df,
            }

            return new_adata

    def _get_processing_strategy(self, fragments: bool | None = None) -> bool:
        """Determine the processing strategy based on fragments and dataset size"""
        if fragments is not None:
            return fragments
        try:
            fragments_list = self.slaf.expression.get_fragments()
            return len(fragments_list) > 1
        except Exception:
            return False

Attributes¶

`layers: LazyLayersView` `property` ¶

Access to layers (dictionary-like interface).

Returns a dictionary-like view of layers that provides access to alternative representations of the expression matrix (e.g., spliced, unspliced, counts). Layers have the same dimensions as X.

Returns:

Type	Description
`LazyLayersView`	LazyLayersView providing dictionary-like access to layers.

Examples:

>>> # Access a layer
>>> slaf_array = SLAFArray("data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> spliced = adata.layers["spliced"]
>>> print(f"Layer shape: {spliced.shape}")
Layer shape: (1000, 20000)

>>> # List available layers
>>> print(list(adata.layers.keys()))
['spliced', 'unspliced']

>>> # Check if layer exists
>>> assert "spliced" in adata.layers

`obs: LazyObsView` `property` ¶

Cell metadata (observations) - mutable view with DataFrame-like and dict-like interfaces.

Returns a view that provides both DataFrame-like and dictionary-like access to cell metadata columns with support for creating, updating, and deleting columns. This view respects cell selectors from parent LazyAnnData.

When accessed with a string key (e.g., obs["cluster"]), returns a pandas Series (AnnData-compatible). When accessed directly, behaves like a DataFrame.

Returns:

Type	Description
`LazyObsView`	LazyObsView providing DataFrame-like and dictionary-like access to obs columns.

Examples:

>>> # DataFrame-like access (AnnData-compatible)
>>> slaf_array = SLAFArray("data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> df = adata.obs  # Returns DataFrame-like view
>>> cluster = adata.obs["cluster"]  # Returns pd.Series (AnnData-compatible)
>>> print(cluster.head())

>>> # Create a new column
>>> adata.obs["new_cluster"] = new_cluster_labels
>>> assert "new_cluster" in adata.obs

>>> # List available columns
>>> print(list(adata.obs.keys()))
['cell_id', 'total_counts', 'cluster', 'new_cluster']

`var: LazyVarView` `property` ¶

Gene metadata (variables) - mutable view with DataFrame-like and dict-like interfaces.

Returns a view that provides both DataFrame-like and dictionary-like access to gene metadata columns with support for creating, updating, and deleting columns. This view respects gene selectors from parent LazyAnnData.

When accessed with a string key (e.g., var["highly_variable"]), returns a pandas Series (AnnData-compatible). When accessed directly, behaves like a DataFrame.

Returns:

Type	Description
`LazyVarView`	LazyVarView providing DataFrame-like and dictionary-like access to var columns.

Examples:

>>> # DataFrame-like access (AnnData-compatible)
>>> slaf_array = SLAFArray("data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> df = adata.var  # Returns DataFrame-like view
>>> hvg = adata.var["highly_variable"]  # Returns pd.Series (AnnData-compatible)
>>> print(hvg.head())

>>> # Create a new column
>>> adata.var["new_annotation"] = new_annotations
>>> assert "new_annotation" in adata.var

`obsm: LazyObsmView` `property` ¶

Multi-dimensional obs annotations (embeddings, PCA, etc.).

Returns a dictionary-like view that provides access to multi-dimensional cell annotations stored as separate columns in cells.lance. Each key maps to a 2D numpy array.

Returns:

Type	Description
`LazyObsmView`	LazyObsmView providing dictionary-like access to obsm keys.

Examples:

>>> # Access an embedding
>>> slaf_array = SLAFArray("data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> umap = adata.obsm["X_umap"]
>>> print(f"UMAP shape: {umap.shape}")
UMAP shape: (1000, 2)

>>> # Create a new embedding
>>> adata.obsm["X_pca"] = pca_coords  # shape: (1000, 50)
>>> assert "X_pca" in adata.obsm

`varm: LazyVarmView` `property` ¶

Multi-dimensional var annotations (gene loadings, etc.).

Returns a dictionary-like view that provides access to multi-dimensional gene annotations stored as separate columns in genes.lance. Each key maps to a 2D numpy array.

Returns:

Type	Description
`LazyVarmView`	LazyVarmView providing dictionary-like access to varm keys.

Examples:

>>> # Access gene loadings
>>> slaf_array = SLAFArray("data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> pcs = adata.varm["PCs"]
>>> print(f"PCs shape: {pcs.shape}")
PCs shape: (20000, 50)

`uns: LazyUnsView` `property` ¶

Unstructured metadata (analysis parameters, etc.).

Returns a dictionary-like view that provides access to unstructured metadata stored in uns.json. Unlike obs/var/obsm/varm, uns metadata is always mutable and stored as JSON.

Returns:

Type	Description
`LazyUnsView`	LazyUnsView providing dictionary-like access to uns keys.

Examples:

>>> # Store metadata
>>> slaf_array = SLAFArray("data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> adata.uns["neighbors"] = {"params": {"n_neighbors": 15}}
>>> print(adata.uns["neighbors"])
{'params': {'n_neighbors': 15}}

`X: LazyExpressionMatrix` `property` ¶

Access to expression data.

Returns the lazy expression matrix that provides scipy.sparse-compatible access to the single-cell expression data. The matrix is lazily evaluated to avoid loading all data into memory.

Returns:

Type	Description
`LazyExpressionMatrix`	LazyExpressionMatrix providing scipy.sparse-compatible interface.

Examples:

>>> # Access expression data
>>> slaf_array = SLAFArray("path/to/data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> print(f"Expression matrix shape: {adata.X.shape}")
Expression matrix shape: (1000, 20000)

>>> # Subsetting expression data
>>> subset_X = adata.X[:100, :5000]
>>> print(f"Subset expression shape: {subset_X.shape}")
Subset expression shape: (100, 5000)

>>> # Check matrix type
>>> print(f"Matrix type: {type(adata.X)}")
Matrix type: <class 'slaf.integrations.anndata.LazyExpressionMatrix'>

`obs_deprecated: pd.DataFrame` `property` ¶

Cell metadata (observations) - DEPRECATED.

.. deprecated:: 0.X This property is deprecated. Use :attr:obs (formerly :attr:obs) instead. This will be removed in a future version.

This property triggers computation of metadata when accessed. For lazy access to metadata structure only, use obs.columns, obs.index, etc.

`var_deprecated: pd.DataFrame` `property` ¶

Gene metadata (variables) - DEPRECATED.

.. deprecated:: 0.X This property is deprecated. Use :attr:var (formerly :attr:var) instead. This will be removed in a future version.

This property triggers computation of metadata when accessed. For lazy access to metadata structure only, use var.columns, var.index, etc.

`obs_names: pd.Index` `property` ¶

Cell names

`var_names: pd.Index` `property` ¶

Gene names

`n_obs: int` `property` ¶

Number of observations (cells)

`n_vars: int` `property` ¶

Number of variables (genes)

`shape: tuple[int, int]` `property` ¶

Get the shape of the data, accounting for any applied filters

Functions¶

`init(slaf_array: SLAFArray, backend: str = 'auto')` ¶

Initialize LazyAnnData with SLAF array.

Parameters:

Name	Type	Description	Default
`slaf_array`	`SLAFArray`	SLAFArray instance containing the single-cell data. Used for database queries and metadata access.	required
`backend`	`str`	Backend for expression matrix. Currently supports "scipy" and "auto". "auto" defaults to "scipy" for sparse matrix operations.	`'auto'`

Raises:

Type	Description
`ValueError`	If the backend is not supported.
`RuntimeError`	If the SLAF array is not properly initialized.

Examples:

>>> # Basic initialization
>>> slaf_array = SLAFArray("path/to/data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> print(f"Backend: {adata.backend}")
Backend: auto

>>> # With explicit backend
>>> adata = LazyAnnData(slaf_array, backend="scipy")
>>> print(f"Backend: {adata.backend}")
Backend: scipy

>>> # Error handling for unsupported backend
>>> try:
...     adata = LazyAnnData(slaf_array, backend="unsupported")
... except ValueError as e:
...     print(f"Error: {e}")
Error: Unknown backend: unsupported

Source code in slaf/integrations/anndata.py

def __init__(
    self,
    slaf_array: SLAFArray,
    backend: str = "auto",
):
    """
    Initialize LazyAnnData with SLAF array.

    Args:
        slaf_array: SLAFArray instance containing the single-cell data.
                   Used for database queries and metadata access.
        backend: Backend for expression matrix. Currently supports "scipy" and "auto".
                "auto" defaults to "scipy" for sparse matrix operations.

    Raises:
        ValueError: If the backend is not supported.
        RuntimeError: If the SLAF array is not properly initialized.

    Examples:
        >>> # Basic initialization
        >>> slaf_array = SLAFArray("path/to/data.slaf")
        >>> adata = LazyAnnData(slaf_array)
        >>> print(f"Backend: {adata.backend}")
        Backend: auto

        >>> # With explicit backend
        >>> adata = LazyAnnData(slaf_array, backend="scipy")
        >>> print(f"Backend: {adata.backend}")
        Backend: scipy

        >>> # Error handling for unsupported backend
        >>> try:
        ...     adata = LazyAnnData(slaf_array, backend="unsupported")
        ... except ValueError as e:
        ...     print(f"Error: {e}")
        Error: Unknown backend: unsupported
    """
    super().__init__()
    self.slaf = slaf_array
    self.backend = backend

    # Initialize expression matrix
    if backend == "scipy" or backend == "auto":
        self._X = LazyExpressionMatrix(slaf_array)
        self._X.parent_adata = self  # Set parent reference for transformations
    else:
        raise ValueError(f"Unknown backend: {backend}")

    # Lazy-loaded metadata
    self._obs = None
    self._var = None
    self._cached_obs_names: pd.Index | None = None
    self._cached_var_names: pd.Index | None = None

    # Filter selectors for subsetting
    self._cell_selector: Any = None
    self._gene_selector: Any = None
    self._filtered_obs: Callable[[], pd.DataFrame] | None = None
    self._filtered_var: Callable[[], pd.DataFrame] | None = None

    # Reference to parent LazyAnnData (for sliced objects)
    self._parent_adata: LazyAnnData | None = None

    # Transformations for lazy evaluation
    self._transformations: dict[str, Any] = {}

    # Layers view (lazy initialization)
    self._layers: LazyLayersView | None = None

`copy() -> LazyAnnData` ¶

Create a copy

Source code in slaf/integrations/anndata.py

def copy(self) -> "LazyAnnData":
    """Create a copy"""
    new_adata = LazyAnnData(self.slaf, backend=self.backend)

    # Copy selectors
    new_adata._cell_selector = self._cell_selector
    new_adata._gene_selector = self._gene_selector

    # Copy filtered metadata functions
    new_adata._filtered_obs = self._filtered_obs
    new_adata._filtered_var = self._filtered_var

    # Copy transformations
    new_adata._transformations = self._transformations.copy()

    # Set up parent reference for the expression matrix
    if isinstance(new_adata._X, LazyExpressionMatrix):
        new_adata._X.parent_adata = new_adata

    return new_adata

`to_memory() -> scipy.sparse.spmatrix` ¶

Load entire matrix into memory

Source code in slaf/integrations/anndata.py

def to_memory(self) -> scipy.sparse.spmatrix:
    """Load entire matrix into memory"""
    return self.get_expression_data()

`write(filename: str)` ¶

Write to h5ad format (would need implementation)

Source code in slaf/integrations/anndata.py

def write(self, filename: str):
    """Write to h5ad format (would need implementation)"""
    raise NotImplementedError("Writing LazyAnnData not yet implemented")

`get_expression_data() -> scipy.sparse.csr_matrix` ¶

Get expression data with any applied filters

Source code in slaf/integrations/anndata.py

def get_expression_data(self) -> scipy.sparse.csr_matrix:
    """Get expression data with any applied filters"""
    if isinstance(self._X, LazyExpressionMatrix):
        # If the LazyExpressionMatrix already has selectors stored, just get the data
        # without passing additional selectors to avoid double-composition
        if self._X._cell_selector is not None or self._X._gene_selector is not None:
            return self._X[:, :].compute()
        elif self._cell_selector is not None or self._gene_selector is not None:
            cell_sel = (
                self._cell_selector
                if self._cell_selector is not None
                else slice(None)
            )
            gene_sel = (
                self._gene_selector
                if self._gene_selector is not None
                else slice(None)
            )
            return self._X[cell_sel, gene_sel].compute()
        else:
            return self._X[:, :].compute()
    else:
        raise NotImplementedError(
            "Dask backend not yet implemented for get_expression_data"
        )

`compute() -> sc.AnnData` ¶

Explicitly compute and return a native AnnData object

Source code in slaf/integrations/anndata.py

def compute(self) -> "sc.AnnData":
    """Explicitly compute and return a native AnnData object"""
    import scanpy as sc

    # Create native AnnData object (get DataFrames from views)
    adata = sc.AnnData(
        X=self._X.compute(),
        obs=self.obs._get_dataframe(),
        var=self.var._get_dataframe(),
    )

    return adata

Functions¶

`read_slaf(filename: str, backend: str = 'auto') -> LazyAnnData` ¶

Read SLAF file as LazyAnnData object

Source code in slaf/integrations/anndata.py

def read_slaf(filename: str, backend: str = "auto") -> LazyAnnData:
    """Read SLAF file as LazyAnnData object"""
    slaf_array = SLAFArray(filename)
    return LazyAnnData(slaf_array, backend=backend)

Scanpy Integration¶

`slaf.integrations.scanpy` ¶

Classes¶

`LazyPreprocessing` ¶

Scanpy preprocessing functions with lazy evaluation

Source code in slaf/integrations/scanpy.py

class LazyPreprocessing:
    """Scanpy preprocessing functions with lazy evaluation"""

    @staticmethod
    def calculate_qc_metrics(
        adata: LazyAnnData,
        percent_top: int | list | None = None,
        log1p: bool = True,
        inplace: bool = True,
    ) -> tuple | None:
        """
        Calculate quality control metrics for cells and genes using lazy evaluation.

        This function computes cell and gene-level QC metrics using SQL aggregation
        for memory efficiency. It calculates metrics like total counts, number of
        genes per cell, and number of cells per gene.

        Args:
            adata: LazyAnnData instance containing the single-cell data.
            percent_top: Number of top genes to consider for percent_top calculation.
                        Currently not implemented in lazy version.
            log1p: Whether to compute log1p-transformed versions of count metrics.
                   Adds log1p_total_counts and log1p_n_genes_by_counts to cell metrics,
                   and log1p_total_counts and log1p_n_cells_by_counts to gene metrics.
            inplace: Whether to modify the adata object in place. Currently not fully
                    implemented in lazy version - returns None when True.

        Returns:
            tuple | None: If inplace=False, returns (cell_qc, gene_qc) where:
                - cell_qc: DataFrame with cell-level QC metrics
                - gene_qc: DataFrame with gene-level QC metrics
                If inplace=True, returns None.

        Raises:
            RuntimeError: If the SLAF array is not properly initialized.

        Examples:
            >>> # Calculate QC metrics
            >>> slaf_array = SLAFArray("path/to/data.slaf")
            >>> adata = LazyAnnData(slaf_array)
            >>> cell_qc, gene_qc = LazyPreprocessing.calculate_qc_metrics(
            ...     adata, inplace=False
            ... )
            >>> print(f"Cell QC shape: {cell_qc.shape}")
            Cell QC shape: (1000, 4)
            >>> print(f"Gene QC shape: {gene_qc.shape}")
            Gene QC shape: (20000, 4)

            >>> # With log1p transformation
            >>> cell_qc, gene_qc = LazyPreprocessing.calculate_qc_metrics(
            ...     adata, log1p=True, inplace=False
            ... )
            >>> print(f"Cell QC columns: {list(cell_qc.columns)}")
            Cell QC columns: ['cell_id', 'n_genes_by_counts', 'total_counts', 'log1p_total_counts', 'log1p_n_genes_by_counts']
        """

        # Calculate cell-level metrics via SQL using simple aggregation (no JOINs)
        cell_qc_sql = """
        SELECT
            e.cell_integer_id,
            COUNT(DISTINCT e.gene_integer_id) as n_genes_by_counts,
            SUM(e.value) as total_counts
        FROM expression e
        GROUP BY e.cell_integer_id
        ORDER BY e.cell_integer_id
        """

        cell_qc = adata.slaf.query(cell_qc_sql)

        # Work with polars DataFrame internally
        cell_qc_pl = cell_qc

        # Map cell_integer_id to cell_id for scanpy compatibility
        if hasattr(adata.slaf, "obs") and adata.slaf.obs is not None:
            # Create mapping from cell_integer_id to cell names
            # Use polars DataFrame to get cell names
            obs_df = adata.slaf.obs
            if "cell_id" in obs_df.columns:
                cell_id_to_name = dict(
                    zip(
                        obs_df["cell_integer_id"].to_list(),
                        obs_df["cell_id"].to_list(),
                        strict=False,
                    )
                )
            else:
                # Fallback: create cell names from integer IDs
                cell_id_to_name = {i: f"cell_{i}" for i in range(len(obs_df))}
            # Use vectorized polars operations for mapping
            # Create mapping DataFrame
            cell_map_df = pl.DataFrame(
                {
                    "cell_integer_id": list(cell_id_to_name.keys()),
                    "cell_id": list(cell_id_to_name.values()),
                }
            )
            # Join with mapping DataFrame
            cell_qc_pl = cell_qc_pl.join(cell_map_df, on="cell_integer_id", how="left")
            # Fill any missing values with default format
            cell_qc_pl = cell_qc_pl.with_columns(
                pl.col("cell_id").fill_null(
                    pl.col("cell_integer_id")
                    .cast(pl.Utf8)
                    .map_elements(lambda x: f"cell_{x}", return_dtype=pl.Utf8)
                )
            )
        else:
            # Fallback: use cell_integer_id as cell_id
            cell_qc_pl = cell_qc_pl.with_columns(
                pl.col("cell_integer_id").cast(pl.Utf8).alias("cell_id")
            )

        # Add log1p transformed counts if requested
        if log1p:
            cell_qc_pl = cell_qc_pl.with_columns(
                [
                    pl.col("total_counts").log1p().alias("log1p_total_counts"),
                    pl.col("n_genes_by_counts")
                    .log1p()
                    .alias("log1p_n_genes_by_counts"),
                ]
            )

        # Convert to pandas for return compatibility
        cell_qc = cell_qc_pl.to_pandas()

        # Calculate gene-level metrics via SQL using simple aggregation (no JOINs)
        gene_qc_sql = """
        SELECT
            e.gene_integer_id,
            COUNT(DISTINCT e.cell_integer_id) AS n_cells_by_counts,
            SUM(e.value) AS total_counts
        FROM expression e
        GROUP BY e.gene_integer_id
        ORDER BY e.gene_integer_id
        """

        gene_qc = adata.slaf.query(gene_qc_sql)

        # Work with polars DataFrames internally
        gene_qc_pl = gene_qc

        # For scanpy compatibility, we need to ensure all genes are present
        # Use in-memory var if available, otherwise fall back to SQL
        if hasattr(adata.slaf, "var") and adata.slaf.var is not None:
            # Use the materialized var metadata directly
            expected_genes_pl = pl.DataFrame(
                {"gene_integer_id": range(len(adata.slaf.var))}
            )
        else:
            expected_genes_sql = """
            SELECT gene_integer_id
            FROM genes
            ORDER BY gene_integer_id
            """
            expected_genes = adata.slaf.query(expected_genes_sql)
            expected_genes_pl = expected_genes

        # Create a complete gene_qc DataFrame with all expected genes using polars
        gene_qc_complete_pl = expected_genes_pl.join(
            gene_qc_pl, on="gene_integer_id", how="left"
        ).fill_null(0)

        # Ensure proper data types
        gene_qc_complete_pl = gene_qc_complete_pl.with_columns(
            [
                pl.col("n_cells_by_counts").cast(pl.Int64),
                pl.col("total_counts").cast(pl.Float64),
            ]
        )

        # Map gene_integer_id to gene names for scanpy compatibility
        if hasattr(adata.slaf, "var") and adata.slaf.var is not None:
            # Create mapping from gene_integer_id to gene names
            # Use polars DataFrame to get gene names
            var_df = adata.slaf.var
            if "gene_id" in var_df.columns:
                gene_id_to_name = dict(
                    zip(
                        var_df["gene_integer_id"].to_list(),
                        var_df["gene_id"].to_list(),
                        strict=False,
                    )
                )
            else:
                # Fallback: create gene names from integer IDs
                gene_id_to_name = {i: f"gene_{i}" for i in range(len(var_df))}
            # Use vectorized polars operations for mapping
            # Create mapping DataFrame
            gene_map_df = pl.DataFrame(
                {
                    "gene_integer_id": list(gene_id_to_name.keys()),
                    "gene_id": list(gene_id_to_name.values()),
                }
            )
            # Join with mapping DataFrame
            gene_qc_complete_pl = gene_qc_complete_pl.join(
                gene_map_df, on="gene_integer_id", how="left"
            )
            # Fill any missing values with default format
            gene_qc_complete_pl = gene_qc_complete_pl.with_columns(
                pl.col("gene_id").fill_null(
                    pl.col("gene_integer_id")
                    .cast(pl.Utf8)
                    .map_elements(lambda x: f"gene_{x}", return_dtype=pl.Utf8)
                )
            )
        else:
            # Fallback: use gene_integer_id as gene_id
            gene_qc_complete_pl = gene_qc_complete_pl.with_columns(
                pl.col("gene_integer_id").cast(pl.Utf8).alias("gene_id")
            )

        # Convert to pandas and set gene_id as index
        gene_qc_complete = gene_qc_complete_pl.to_pandas().set_index("gene_id")

        if log1p:
            gene_qc_complete["log1p_total_counts"] = np.log1p(
                gene_qc_complete["total_counts"]
            )
            gene_qc_complete["log1p_n_cells_by_counts"] = np.log1p(
                gene_qc_complete["n_cells_by_counts"]
            )

        if inplace:
            # Update the metadata tables in SLAF
            # This would require implementing metadata updates in SLAF
            # For now, just update the cached obs/var

            # Update obs
            adata._obs = None  # Clear cache
            for _ in cell_qc.iterrows():
                # Would need to implement metadata updates
                pass

            # Update var
            adata._var = None  # Clear cache
            for _ in gene_qc_complete.iterrows():
                # Would need to implement metadata updates
                pass

            return None
        else:
            return cell_qc, gene_qc_complete

    @staticmethod
    def filter_cells(
        adata: LazyAnnData,
        min_counts: int | None = None,
        min_genes: int | None = None,
        max_counts: int | None = None,
        max_genes: int | None = None,
        inplace: bool = True,
    ) -> LazyAnnData | None:
        """
        Filter cells based on quality control metrics using lazy evaluation.

        This function filters cells based on their total counts and number of genes
        using SQL aggregation for memory efficiency. It supports both minimum and
        maximum thresholds for each metric.

        Args:
            adata: LazyAnnData instance containing the single-cell data.
            min_counts: Minimum total counts per cell. Cells with fewer counts are filtered.
            min_genes: Minimum number of genes per cell. Cells with fewer genes are filtered.
            max_counts: Maximum total counts per cell. Cells with more counts are filtered.
            max_genes: Maximum number of genes per cell. Cells with more genes are filtered.
            inplace: Whether to modify the adata object in place. Currently not fully
                    implemented in lazy version - returns None when True.

        Returns:
            LazyAnnData | None: If inplace=False, returns filtered LazyAnnData.
                               If inplace=True, returns None.

        Raises:
            ValueError: If all cells are filtered out by the criteria.
            RuntimeError: If the SLAF array is not properly initialized.

        Examples:
            >>> # Filter cells with basic criteria
            >>> slaf_array = SLAFArray("path/to/data.slaf")
            >>> adata = LazyAnnData(slaf_array)
            >>> filtered = LazyPreprocessing.filter_cells(
            ...     adata, min_counts=100, min_genes=50, inplace=False
            ... )
            >>> print(f"Original cells: {adata.n_obs}")
            Original cells: 1000
            >>> print(f"Filtered cells: {filtered.n_obs}")
            Filtered cells: 850

            >>> # Filter with maximum thresholds
            >>> filtered = LazyPreprocessing.filter_cells(
            ...     adata, max_counts=10000, max_genes=5000, inplace=False
            ... )
            >>> print(f"Cells after max filtering: {filtered.n_obs}")
            Cells after max filtering: 920

            >>> # Error when all cells filtered out
            >>> try:
            ...     LazyPreprocessing.filter_cells(adata, min_counts=1000000)
            ... except ValueError as e:
            ...     print(f"Error: {e}")
            Error: All cells were filtered out
        """

        # Build filter conditions
        conditions = []

        if min_counts is not None:
            conditions.append(f"cell_stats.total_counts >= {min_counts}")
        if max_counts is not None:
            conditions.append(f"cell_stats.total_counts <= {max_counts}")
        if min_genes is not None:
            conditions.append(f"cell_stats.n_genes_by_counts >= {min_genes}")
        if max_genes is not None:
            conditions.append(f"cell_stats.n_genes_by_counts <= {max_genes}")

        if not conditions:
            return adata if not inplace else None

        where_clause = " AND ".join(conditions)

        # Get filtered cell IDs using simple aggregation (no JOINs)
        filter_sql = f"""
        SELECT cell_stats.cell_integer_id
        FROM (
            SELECT
                e.cell_integer_id,
                COUNT(DISTINCT e.gene_integer_id) as n_genes_by_counts,
                SUM(e.value) as total_counts
            FROM expression e
            GROUP BY e.cell_integer_id
        ) cell_stats
        WHERE ({where_clause})
        ORDER BY cell_stats.cell_integer_id
        """

        filtered_cells = adata.slaf.query(filter_sql)

        if len(filtered_cells) == 0:
            raise ValueError("All cells were filtered out")

        # Create boolean mask from the filtered cell IDs
        # Use the materialized obs metadata to map cell_integer_id to cell names
        if hasattr(adata.slaf, "obs") and adata.slaf.obs is not None:
            # Get all cell integer IDs that pass the filter
            filtered_cell_ids = set(filtered_cells["cell_integer_id"].to_list())

            # Create boolean mask for all cells
            all_cell_ids = adata.slaf.obs["cell_integer_id"].to_list()
            boolean_mask = [cell_id in filtered_cell_ids for cell_id in all_cell_ids]

            # Actually subset the LazyAnnData object (like scanpy does)
            if inplace:
                # Set up the filtered obs function to apply the boolean mask
                def filtered_obs() -> pd.DataFrame:
                    obs_df = adata.slaf.obs
                    if obs_df is not None:
                        # Work with polars DataFrame internally
                        obs_pl = obs_df
                        # Drop cell_integer_id column if present
                        if "cell_integer_id" in obs_pl.columns:
                            obs_pl = obs_pl.drop("cell_integer_id")
                        # Apply boolean mask
                        obs_pl = obs_pl.filter(pl.Series(boolean_mask))
                        # Convert to pandas DataFrame for AnnData compatibility
                        obs_copy = obs_pl.to_pandas()
                        # Set cell_id as index if present
                        if "cell_id" in obs_copy.columns:
                            obs_copy = obs_copy.set_index("cell_id")
                        # Set index name to match AnnData format
                        if hasattr(obs_copy, "index"):
                            obs_copy.index.name = "cell_id"
                        return obs_copy
                    return pd.DataFrame()

                # Set the filtered functions
                adata._filtered_obs = filtered_obs
                adata._filtered_var = None  # No gene filtering
                adata._cell_selector = boolean_mask
                adata._gene_selector = None

                # Clear cached names to force recalculation
                adata._cached_obs_names = None
                adata._cached_var_names = None

                return None
            else:
                # Create a new filtered LazyAnnData
                filtered_adata = LazyAnnData(adata.slaf, backend=adata.backend)

                # Set up the filtered obs function
                def filtered_obs() -> pd.DataFrame:
                    obs_df = adata.slaf.obs
                    if obs_df is not None:
                        # Work with polars DataFrame internally
                        obs_pl = obs_df
                        # Drop cell_integer_id column if present
                        if "cell_integer_id" in obs_pl.columns:
                            obs_pl = obs_pl.drop("cell_integer_id")
                        # Apply boolean mask
                        obs_pl = obs_pl.filter(pl.Series(boolean_mask))
                        # Convert to pandas DataFrame for AnnData compatibility
                        obs_copy = obs_pl.to_pandas()
                        # Set cell_id as index if present
                        if "cell_id" in obs_copy.columns:
                            obs_copy = obs_copy.set_index("cell_id")
                        # Set index name to match AnnData format
                        if hasattr(obs_copy, "index"):
                            obs_copy.index.name = "cell_id"
                        return obs_copy
                    return pd.DataFrame()

                # Set the filtered functions
                filtered_adata._filtered_obs = filtered_obs
                filtered_adata._filtered_var = None  # No gene filtering
                filtered_adata._cell_selector = boolean_mask
                filtered_adata._gene_selector = None

                return filtered_adata
        else:
            # Fallback: create boolean mask by mapping integer IDs to cell names
            # obs_names contains string cell IDs, so we need to map integer IDs to cell names
            if hasattr(adata.slaf, "obs") and adata.slaf.obs is not None:
                # Create mapping from cell_integer_id to cell_id
                obs_df = adata.slaf.obs
                if "cell_id" in obs_df.columns and "cell_integer_id" in obs_df.columns:
                    id_to_name = dict(
                        zip(
                            obs_df["cell_integer_id"].to_list(),
                            obs_df["cell_id"].to_list(),
                            strict=False,
                        )
                    )
                    # Map filtered integer IDs to cell names
                    filtered_cell_names = {
                        id_to_name.get(cid)
                        for cid in filtered_cells["cell_integer_id"].to_list()
                        if id_to_name.get(cid) is not None
                    }
                    # Create boolean mask
                    cell_mask = adata.obs_names.isin(filtered_cell_names)
                else:
                    # Fallback: use integer IDs directly (assume obs_names are integer strings)
                    filtered_cell_ids = {
                        str(cid) for cid in filtered_cells["cell_integer_id"].to_list()
                    }
                    cell_mask = adata.obs_names.astype(str).isin(filtered_cell_ids)
            else:
                # Last resort: assume obs_names match integer IDs as strings
                filtered_cell_ids = {
                    str(cid) for cid in filtered_cells["cell_integer_id"].to_list()
                }
                cell_mask = adata.obs_names.astype(str).isin(filtered_cell_ids)

        if inplace:
            # Apply filter to adata (would need proper implementation)
            # For now, just return the original adata
            print(
                f"Filtered out {np.sum(~cell_mask)} cells, {np.sum(cell_mask)} remaining"
            )
            return None
        else:
            # Create new filtered LazyAnnData
            filtered_adata = adata.copy()
            # Apply mask (would need proper implementation)
            return filtered_adata

    @staticmethod
    def filter_genes(
        adata: LazyAnnData,
        min_counts: int | None = None,
        min_cells: int | None = None,
        max_counts: int | None = None,
        max_cells: int | None = None,
        inplace: bool = True,
    ) -> LazyAnnData | None:
        """
        Filter genes based on quality control metrics using lazy evaluation.

        This function filters genes based on their total counts and number of cells
        using SQL aggregation for memory efficiency. It supports both minimum and
        maximum thresholds for each metric.

        Args:
            adata: LazyAnnData instance containing the single-cell data.
            min_counts: Minimum total counts per gene. Genes with fewer counts are filtered.
            min_cells: Minimum number of cells per gene. Genes expressed in fewer cells are filtered.
            max_counts: Maximum total counts per gene. Genes with more counts are filtered.
            max_cells: Maximum number of cells per gene. Genes expressed in more cells are filtered.
            inplace: Whether to modify the adata object in place. Currently not fully
                    implemented in lazy version - returns None when True.

        Returns:
            LazyAnnData | None: If inplace=False, returns filtered LazyAnnData.
                               If inplace=True, returns None.

        Raises:
            ValueError: If all genes are filtered out by the criteria.
            RuntimeError: If the SLAF array is not properly initialized.

        Examples:
            >>> # Filter genes with basic criteria
            >>> slaf_array = SLAFArray("path/to/data.slaf")
            >>> adata = LazyAnnData(slaf_array)
            >>> filtered = LazyPreprocessing.filter_genes(
            ...     adata, min_counts=10, min_cells=5, inplace=False
            ... )
            >>> print(f"Original genes: {adata.n_vars}")
            Original genes: 20000
            >>> print(f"Filtered genes: {filtered.n_vars}")
            Filtered genes: 15000

            >>> # Filter with maximum thresholds
            >>> filtered = LazyPreprocessing.filter_genes(
            ...     adata, max_counts=100000, max_cells=1000, inplace=False
            ... )
            >>> print(f"Genes after max filtering: {filtered.n_vars}")
            Genes after max filtering: 18000

            >>> # Error when all genes filtered out
            >>> try:
            ...     LazyPreprocessing.filter_genes(adata, min_counts=1000000)
            ... except ValueError as e:
            ...     print(f"Error: {e}")
            Error: All genes were filtered out
        """

        # Build filter conditions for genes
        conditions = []

        if min_counts is not None:
            conditions.append(f"gene_stats.total_counts >= {min_counts}")
        if max_counts is not None:
            conditions.append(f"gene_stats.total_counts <= {max_counts}")
        if min_cells is not None:
            conditions.append(f"gene_stats.n_cells_by_counts >= {min_cells}")
        if max_cells is not None:
            conditions.append(f"gene_stats.n_cells_by_counts <= {max_cells}")

        if not conditions:
            return adata if not inplace else None

        where_clause = " AND ".join(conditions)

        # Get filtered gene IDs using simple aggregation (no JOINs)
        filter_sql = f"""
        SELECT gene_stats.gene_integer_id
        FROM (
            SELECT
                e.gene_integer_id,
                COUNT(DISTINCT e.cell_integer_id) AS n_cells_by_counts,
                SUM(e.value) AS total_counts
            FROM expression e
            GROUP BY e.gene_integer_id
        ) gene_stats
        WHERE {where_clause}
        ORDER BY gene_stats.gene_integer_id
        """

        filtered_genes = adata.slaf.query(filter_sql)

        if len(filtered_genes) == 0:
            raise ValueError("All genes were filtered out")

        # Create boolean mask from the filtered gene IDs
        # Use the materialized var metadata to map gene_integer_id to gene names
        if hasattr(adata.slaf, "var") and adata.slaf.var is not None:
            # Create a mapping from gene_integer_id to gene names
            # Use polars DataFrame to get gene names
            var_df = adata.slaf.var
            if "gene_id" in var_df.columns:
                gene_id_to_name = dict(
                    zip(
                        var_df["gene_integer_id"].to_list(),
                        var_df["gene_id"].to_list(),
                        strict=False,
                    )
                )
            else:
                # Fallback: create gene names from integer IDs
                gene_id_to_name = {i: f"gene_{i}" for i in range(len(var_df))}
            filtered_gene_names = [
                gene_id_to_name.get(gid, f"gene_{gid}")
                for gid in filtered_genes["gene_integer_id"]
            ]
            gene_mask = adata.var_names.isin(filtered_gene_names)
        else:
            # Fallback to using gene_integer_id directly
            gene_mask = adata.var_names.isin(filtered_genes["gene_integer_id"])

        if inplace:
            # Apply filter to adata (would need proper implementation)
            print(
                f"Filtered out {np.sum(~gene_mask)} genes, {np.sum(gene_mask)} remaining"
            )
            return None
        else:
            # Create new filtered LazyAnnData
            filtered_adata = adata.copy()
            # Apply mask (would need proper implementation)
            return filtered_adata

    @staticmethod
    def normalize_total(
        adata: LazyAnnData,
        target_sum: float | None = 1e4,
        inplace: bool = True,
        fragments: bool | None = None,
    ) -> LazyAnnData | None:
        """
        Normalize counts per cell to target sum using lazy evaluation.

        This function normalizes the expression data so that each cell has a total
        count equal to target_sum. The normalization is applied lazily and stored
        as a transformation that will be computed when the data is accessed.

        Args:
            adata: LazyAnnData instance containing the single-cell data.
            target_sum: Target sum for normalization. Default is 10,000.
            inplace: Whether to modify the adata object in place. If False, returns
                    a copy with the transformation applied.
            fragments: Whether to use fragment-based processing. If None, automatically
                      selects based on dataset characteristics.

        Returns:
            LazyAnnData | None: If inplace=False, returns LazyAnnData with transformation.
                               If inplace=True, returns None.

        Raises:
            ValueError: If target_sum is not positive.
            RuntimeError: If the SLAF array is not properly initialized.

        Examples:
            >>> # Basic normalization
            >>> slaf_array = SLAFArray("path/to/data.slaf")
            >>> adata = LazyAnnData(slaf_array)
            >>> LazyPreprocessing.normalize_total(adata, target_sum=10000)
            Applied normalize_total with target_sum=10000

            >>> # Custom target sum
            >>> adata_copy = adata.copy()
            >>> LazyPreprocessing.normalize_total(
            ...     adata_copy, target_sum=5000, inplace=False
            ... )
            >>> print("Normalization applied to copy")

            >>> # Error with invalid target_sum
            >>> try:
            ...     LazyPreprocessing.normalize_total(adata, target_sum=0)
            ... except ValueError as e:
            ...     print(f"Error: {e}")
            Error: target_sum must be positive
        """
        if target_sum is None:
            target_sum = 1e4

        # Validate target_sum
        if target_sum <= 0:
            raise ValueError("target_sum must be positive")

        # Determine processing strategy
        if fragments is not None:
            use_fragments = fragments
        else:
            # Check if dataset has multiple fragments
            try:
                fragments_list = adata.slaf.expression.get_fragments()
                use_fragments = len(fragments_list) > 1
            except Exception:
                use_fragments = False

        if use_fragments:
            # Use fragment-based processing
            try:
                from slaf.core.fragment_processor import FragmentProcessor

                # Get selectors from the LazyAnnData if it's sliced
                cell_selector = getattr(adata, "_cell_selector", None)
                gene_selector = getattr(adata, "_gene_selector", None)

                processor = FragmentProcessor(
                    adata.slaf,
                    cell_selector=cell_selector,
                    gene_selector=gene_selector,
                    max_workers=4,
                    enable_caching=True,
                )
                # Use smart strategy selection for optimal performance
                lazy_pipeline = processor.build_lazy_pipeline_smart(
                    "normalize_total", target_sum=target_sum
                )
                result_df = processor.compute(lazy_pipeline)

                # Update adata with normalized values
                return adata._update_with_normalized_data(
                    result_df, target_sum, inplace
                )

            except Exception as e:
                print(
                    f"Fragment processing failed, falling back to global processing: {e}"
                )
                # Fall back to global processing
                use_fragments = False

        if not use_fragments:
            # Use global processing (original implementation)
            # Get cell totals for normalization using only the expression table
            cell_totals_sql = """
            SELECT
                e.cell_integer_id,
                SUM(e.value) as total_counts
            FROM expression e
            GROUP BY e.cell_integer_id
            ORDER BY e.cell_integer_id
            """

            cell_totals = adata.slaf.query(cell_totals_sql)

        # Work with polars DataFrame internally
        cell_totals_pl = cell_totals

        # Create normalization factors using polars
        # Map cell_integer_id to cell names for compatibility with anndata.py
        if hasattr(adata.slaf, "obs") and adata.slaf.obs is not None:
            # Create mapping from cell_integer_id to cell names
            # Use polars DataFrame to get cell names
            obs_df = adata.slaf.obs
            if "cell_id" in obs_df.columns:
                cell_id_to_name = dict(
                    zip(
                        obs_df["cell_integer_id"].to_list(),
                        obs_df["cell_id"].to_list(),
                        strict=False,
                    )
                )
            else:
                # Fallback: create cell names from integer IDs
                cell_id_to_name = {i: f"cell_{i}" for i in range(len(obs_df))}
            # Use vectorized polars operations for mapping
            # Create mapping DataFrame
            cell_map_df = pl.DataFrame(
                {
                    "cell_integer_id": list(cell_id_to_name.keys()),
                    "cell_id": list(cell_id_to_name.values()),
                }
            )
            # Join with mapping DataFrame
            cell_totals_pl = cell_totals_pl.join(
                cell_map_df, on="cell_integer_id", how="left"
            )
            # Fill any missing values with default format
            cell_totals_pl = cell_totals_pl.with_columns(
                [
                    pl.col("cell_id").fill_null(
                        pl.col("cell_integer_id")
                        .cast(pl.Utf8)
                        .map_elements(lambda x: f"cell_{x}", return_dtype=pl.Utf8)
                    ),
                    (target_sum / pl.col("total_counts")).alias("normalization_factor"),
                ]
            )
            # Convert to dictionary for compatibility
            normalization_dict = dict(
                zip(
                    cell_totals_pl["cell_id"].to_list(),
                    cell_totals_pl["normalization_factor"].to_list(),
                    strict=False,
                )
            )
        else:
            # Fallback: use cell_integer_id as string keys
            cell_totals_pl = cell_totals_pl.with_columns(
                [
                    pl.col("cell_integer_id").cast(pl.Utf8).alias("cell_id"),
                    (target_sum / pl.col("total_counts")).alias("normalization_factor"),
                ]
            )
            # Convert to dictionary for compatibility
            normalization_dict = dict(
                zip(
                    cell_totals_pl["cell_id"].to_list(),
                    cell_totals_pl["normalization_factor"].to_list(),
                    strict=False,
                )
            )

        if inplace:
            # Store normalization factors for lazy application
            if not hasattr(adata, "_transformations"):
                adata._transformations = {}

            adata._transformations["normalize_total"] = {
                "type": "normalize_total",
                "target_sum": float(
                    f"{target_sum:.10f}"
                ),  # Convert to regular decimal format
                "cell_factors": normalization_dict,
            }

            print(f"Applied normalize_total with target_sum={target_sum}")
            return None
        else:
            # Create a copy with the transformation (copy-on-write)
            new_adata = adata.copy()
            if not hasattr(new_adata, "_transformations"):
                new_adata._transformations = {}

            new_adata._transformations["normalize_total"] = {
                "type": "normalize_total",
                "target_sum": float(
                    f"{target_sum:.10f}"
                ),  # Convert to regular decimal format
                "cell_factors": normalization_dict,
            }

            return new_adata

    @staticmethod
    def log1p(
        adata: LazyAnnData, inplace: bool = True, fragments: bool | None = None
    ) -> LazyAnnData | None:
        """
        Apply log1p transformation to expression data using lazy evaluation.

        This function applies log(1 + x) transformation to the expression data.
        The transformation is stored lazily and will be computed when the data
        is accessed, avoiding memory-intensive operations on large datasets.

        Args:
            adata: LazyAnnData instance containing the single-cell data.
            inplace: Whether to modify the adata object in place. If False, returns
                    a copy with the transformation applied.

        Returns:
            LazyAnnData | None: If inplace=False, returns LazyAnnData with transformation.
                               If inplace=True, returns None.

        Raises:
            RuntimeError: If the SLAF array is not properly initialized.

        Examples:
            >>> # Apply log1p transformation
            >>> slaf_array = SLAFArray("path/to/data.slaf")
            >>> adata = LazyAnnData(slaf_array)
            >>> LazyPreprocessing.log1p(adata)
            Applied log1p transformation

            >>> # Apply to copy
            >>> adata_copy = adata.copy()
            >>> LazyPreprocessing.log1p(adata_copy, inplace=False)
            >>> print("Log1p transformation applied to copy")

            >>> # Check transformation was stored
            >>> print("log1p" in adata._transformations)
            True
        """
        # Determine processing strategy
        if fragments is not None:
            use_fragments = fragments
        else:
            # Check if dataset has multiple fragments
            try:
                fragments_list = adata.slaf.expression.get_fragments()
                use_fragments = len(fragments_list) > 1
            except Exception:
                use_fragments = False

        if use_fragments:
            # Use fragment-based processing
            try:
                from slaf.core.fragment_processor import FragmentProcessor

                # Get selectors from the LazyAnnData if it's sliced
                cell_selector = getattr(adata, "_cell_selector", None)
                gene_selector = getattr(adata, "_gene_selector", None)

                processor = FragmentProcessor(
                    adata.slaf,
                    cell_selector=cell_selector,
                    gene_selector=gene_selector,
                    max_workers=4,
                    enable_caching=True,
                )
                # Use smart strategy selection for optimal performance
                lazy_pipeline = processor.build_lazy_pipeline_smart("log1p")
                result_df = processor.compute(lazy_pipeline)

                # Update adata with log1p values
                return adata._update_with_log1p_data(result_df, inplace)

            except Exception as e:
                print(
                    f"Fragment processing failed, falling back to global processing: {e}"
                )
                # Fall back to global processing
                use_fragments = False

        if not use_fragments:
            # Use global processing (original implementation)
            if inplace:
                # Store log1p transformation for lazy application
                if not hasattr(adata, "_transformations"):
                    adata._transformations = {}

                adata._transformations["log1p"] = {"type": "log1p", "applied": True}

                print("Applied log1p transformation")
                return None
            else:
                # Create a copy with the transformation (copy-on-write)
                new_adata = adata.copy()
                if not hasattr(new_adata, "_transformations"):
                    new_adata._transformations = {}

                new_adata._transformations["log1p"] = {"type": "log1p", "applied": True}

                return new_adata

    @staticmethod
    def highly_variable_genes(
        adata: LazyAnnData,
        min_mean: float = 0.0125,
        max_mean: float = 3,
        min_disp: float = 0.5,
        max_disp: float = np.inf,
        n_top_genes: int | None = None,
        inplace: bool = True,
    ) -> pd.DataFrame | None:
        """
        Identify highly variable genes using lazy evaluation.

        This function identifies genes with high cell-to-cell variation in expression
        using SQL aggregation for memory efficiency. It calculates mean expression
        and dispersion for each gene and applies filtering criteria.

        Args:
            adata: LazyAnnData instance containing the single-cell data.
            min_mean: Minimum mean expression for genes to be considered.
            max_mean: Maximum mean expression for genes to be considered.
            min_disp: Minimum dispersion for genes to be considered.
            max_disp: Maximum dispersion for genes to be considered.
            n_top_genes: Number of top genes to select by dispersion.
                        If specified, overrides min_disp and max_disp criteria.
            inplace: Whether to modify the adata object in place. Currently not
                    fully implemented - returns None when True.

        Returns:
            pd.DataFrame | None: If inplace=False, returns DataFrame with gene statistics
                                and highly_variable column. If inplace=True, returns None.

        Raises:
            RuntimeError: If the SLAF array is not properly initialized.

        Examples:
            >>> # Find highly variable genes
            >>> slaf_array = SLAFArray("path/to/data.slaf")
            >>> adata = LazyAnnData(slaf_array)
            >>> hvg_stats = LazyPreprocessing.highly_variable_genes(
            ...     adata, inplace=False
            ... )
            >>> print(f"Highly variable genes: {hvg_stats['highly_variable'].sum()}")
            Highly variable genes: 1500

            >>> # With custom criteria
            >>> hvg_stats = LazyPreprocessing.highly_variable_genes(
            ...     adata, min_mean=0.1, max_mean=5, min_disp=1.0, inplace=False
            ... )
            >>> print(f"Genes meeting criteria: {hvg_stats['highly_variable'].sum()}")
            Genes meeting criteria: 800

            >>> # Select top N genes
            >>> hvg_stats = LazyPreprocessing.highly_variable_genes(
            ...     adata, n_top_genes=1000, inplace=False
            ... )
            >>> print(f"Top genes selected: {hvg_stats['highly_variable'].sum()}")
            Top genes selected: 1000
        """

        # Calculate gene statistics via SQL using simple aggregation (no JOINs)
        stats_sql = """
        SELECT
            e.gene_integer_id,
            COUNT(DISTINCT e.cell_integer_id) AS n_cells,
            AVG(e.value) AS mean_expr,
            VARIANCE(e.value) AS variance,
            CASE WHEN AVG(e.value) > 0 THEN VARIANCE(e.value) / AVG(e.value) ELSE 0 END as dispersion
        FROM expression e
        GROUP BY e.gene_integer_id
        ORDER BY e.gene_integer_id
        """

        gene_stats = adata.slaf.query(stats_sql)

        # Work with polars DataFrames internally
        gene_stats_pl = gene_stats

        # Get the expected gene_integer_ids from the materialized var metadata
        # Use in-memory var if available, otherwise fall back to SQL
        if hasattr(adata.slaf, "var") and adata.slaf.var is not None:
            # Use the materialized var metadata directly
            expected_genes_pl = pl.DataFrame(
                {"gene_integer_id": range(len(adata.slaf.var))}
            )
        else:
            # Fallback to SQL if var is not available
            expected_genes_sql = """
            SELECT gene_integer_id
            FROM genes
            ORDER BY gene_integer_id
            """
            expected_genes = adata.slaf.query(expected_genes_sql)
            expected_genes_pl = expected_genes

        # Create a complete gene_stats DataFrame with all expected genes using polars
        gene_stats_complete_pl = expected_genes_pl.join(
            gene_stats_pl, on="gene_integer_id", how="left"
        ).fill_null(0)

        # Ensure proper data types
        gene_stats_complete_pl = gene_stats_complete_pl.with_columns(
            [
                pl.col("n_cells").cast(pl.Int64),
                pl.col("mean_expr").cast(pl.Float64),
                pl.col("variance").cast(pl.Float64),
                pl.col("dispersion").cast(pl.Float64),
            ]
        )

        # Map gene_integer_id to gene_id for scanpy compatibility
        if hasattr(adata.slaf, "var") and adata.slaf.var is not None:
            # Create mapping from gene_integer_id to gene names
            # Use polars DataFrame to get gene names
            var_df = adata.slaf.var
            if "gene_id" in var_df.columns:
                gene_id_to_name = dict(
                    zip(
                        var_df["gene_integer_id"].to_list(),
                        var_df["gene_id"].to_list(),
                        strict=False,
                    )
                )
            else:
                # Fallback: create gene names from integer IDs
                gene_id_to_name = {i: f"gene_{i}" for i in range(len(var_df))}
            # Use vectorized polars operations for mapping
            # Create mapping DataFrame
            gene_map_df = pl.DataFrame(
                {
                    "gene_integer_id": list(gene_id_to_name.keys()),
                    "gene_id": list(gene_id_to_name.values()),
                }
            )
            # Join with mapping DataFrame
            gene_stats_complete_pl = gene_stats_complete_pl.join(
                gene_map_df, on="gene_integer_id", how="left"
            )
            # Fill any missing values with default format
            gene_stats_complete_pl = gene_stats_complete_pl.with_columns(
                pl.col("gene_id").fill_null(
                    pl.col("gene_integer_id")
                    .cast(pl.Utf8)
                    .map_elements(lambda x: f"gene_{x}", return_dtype=pl.Utf8)
                )
            )
        else:
            # Fallback: use gene_integer_id as gene_id
            gene_stats_complete_pl = gene_stats_complete_pl.with_columns(
                pl.col("gene_integer_id").cast(pl.Utf8).alias("gene_id")
            )

        # Convert to pandas and set gene_id as index for scanpy compatibility
        gene_stats_complete = gene_stats_complete_pl.to_pandas().set_index("gene_id")

        # Apply HVG criteria
        hvg_mask = (
            (gene_stats_complete["mean_expr"] >= min_mean)
            & (gene_stats_complete["mean_expr"] <= max_mean)
            & (gene_stats_complete["dispersion"] >= min_disp)
            & (gene_stats_complete["dispersion"] <= max_disp)
        )

        gene_stats_complete["highly_variable"] = hvg_mask

        if n_top_genes is not None:
            # Select top N genes by dispersion
            top_genes = gene_stats_complete.nlargest(n_top_genes, "dispersion")
            gene_stats_complete["highly_variable"] = gene_stats_complete.index.isin(
                top_genes.index
            )

        if inplace:
            # Update var metadata (would need implementation)
            print(f"Identified {hvg_mask.sum()} highly variable genes")
            return None
        else:
            return gene_stats_complete

Functions¶

`calculate_qc_metrics(adata: LazyAnnData, percent_top: int | list | None = None, log1p: bool = True, inplace: bool = True) -> tuple | None` `staticmethod` ¶

Calculate quality control metrics for cells and genes using lazy evaluation.

This function computes cell and gene-level QC metrics using SQL aggregation for memory efficiency. It calculates metrics like total counts, number of genes per cell, and number of cells per gene.

Parameters:

Name	Type	Description	Default
`adata`	`LazyAnnData`	LazyAnnData instance containing the single-cell data.	required
`percent_top`	`int \| list \| None`	Number of top genes to consider for percent_top calculation. Currently not implemented in lazy version.	`None`
`log1p`	`bool`	Whether to compute log1p-transformed versions of count metrics. Adds log1p_total_counts and log1p_n_genes_by_counts to cell metrics, and log1p_total_counts and log1p_n_cells_by_counts to gene metrics.	`True`
`inplace`	`bool`	Whether to modify the adata object in place. Currently not fully implemented in lazy version - returns None when True.	`True`

Returns:

Type	Description
`tuple \| None`	tuple \| None: If inplace=False, returns (cell_qc, gene_qc) where: - cell_qc: DataFrame with cell-level QC metrics - gene_qc: DataFrame with gene-level QC metrics If inplace=True, returns None.

Raises:

Type	Description
`RuntimeError`	If the SLAF array is not properly initialized.

Examples:

>>> # Calculate QC metrics
>>> slaf_array = SLAFArray("path/to/data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> cell_qc, gene_qc = LazyPreprocessing.calculate_qc_metrics(
...     adata, inplace=False
... )
>>> print(f"Cell QC shape: {cell_qc.shape}")
Cell QC shape: (1000, 4)
>>> print(f"Gene QC shape: {gene_qc.shape}")
Gene QC shape: (20000, 4)

>>> # With log1p transformation
>>> cell_qc, gene_qc = LazyPreprocessing.calculate_qc_metrics(
...     adata, log1p=True, inplace=False
... )
>>> print(f"Cell QC columns: {list(cell_qc.columns)}")
Cell QC columns: ['cell_id', 'n_genes_by_counts', 'total_counts', 'log1p_total_counts', 'log1p_n_genes_by_counts']

Source code in slaf/integrations/scanpy.py

@staticmethod
def calculate_qc_metrics(
    adata: LazyAnnData,
    percent_top: int | list | None = None,
    log1p: bool = True,
    inplace: bool = True,
) -> tuple | None:
    """
    Calculate quality control metrics for cells and genes using lazy evaluation.

    This function computes cell and gene-level QC metrics using SQL aggregation
    for memory efficiency. It calculates metrics like total counts, number of
    genes per cell, and number of cells per gene.

    Args:
        adata: LazyAnnData instance containing the single-cell data.
        percent_top: Number of top genes to consider for percent_top calculation.
                    Currently not implemented in lazy version.
        log1p: Whether to compute log1p-transformed versions of count metrics.
               Adds log1p_total_counts and log1p_n_genes_by_counts to cell metrics,
               and log1p_total_counts and log1p_n_cells_by_counts to gene metrics.
        inplace: Whether to modify the adata object in place. Currently not fully
                implemented in lazy version - returns None when True.

    Returns:
        tuple | None: If inplace=False, returns (cell_qc, gene_qc) where:
            - cell_qc: DataFrame with cell-level QC metrics
            - gene_qc: DataFrame with gene-level QC metrics
            If inplace=True, returns None.

    Raises:
        RuntimeError: If the SLAF array is not properly initialized.

    Examples:
        >>> # Calculate QC metrics
        >>> slaf_array = SLAFArray("path/to/data.slaf")
        >>> adata = LazyAnnData(slaf_array)
        >>> cell_qc, gene_qc = LazyPreprocessing.calculate_qc_metrics(
        ...     adata, inplace=False
        ... )
        >>> print(f"Cell QC shape: {cell_qc.shape}")
        Cell QC shape: (1000, 4)
        >>> print(f"Gene QC shape: {gene_qc.shape}")
        Gene QC shape: (20000, 4)

        >>> # With log1p transformation
        >>> cell_qc, gene_qc = LazyPreprocessing.calculate_qc_metrics(
        ...     adata, log1p=True, inplace=False
        ... )
        >>> print(f"Cell QC columns: {list(cell_qc.columns)}")
        Cell QC columns: ['cell_id', 'n_genes_by_counts', 'total_counts', 'log1p_total_counts', 'log1p_n_genes_by_counts']
    """

    # Calculate cell-level metrics via SQL using simple aggregation (no JOINs)
    cell_qc_sql = """
    SELECT
        e.cell_integer_id,
        COUNT(DISTINCT e.gene_integer_id) as n_genes_by_counts,
        SUM(e.value) as total_counts
    FROM expression e
    GROUP BY e.cell_integer_id
    ORDER BY e.cell_integer_id
    """

    cell_qc = adata.slaf.query(cell_qc_sql)

    # Work with polars DataFrame internally
    cell_qc_pl = cell_qc

    # Map cell_integer_id to cell_id for scanpy compatibility
    if hasattr(adata.slaf, "obs") and adata.slaf.obs is not None:
        # Create mapping from cell_integer_id to cell names
        # Use polars DataFrame to get cell names
        obs_df = adata.slaf.obs
        if "cell_id" in obs_df.columns:
            cell_id_to_name = dict(
                zip(
                    obs_df["cell_integer_id"].to_list(),
                    obs_df["cell_id"].to_list(),
                    strict=False,
                )
            )
        else:
            # Fallback: create cell names from integer IDs
            cell_id_to_name = {i: f"cell_{i}" for i in range(len(obs_df))}
        # Use vectorized polars operations for mapping
        # Create mapping DataFrame
        cell_map_df = pl.DataFrame(
            {
                "cell_integer_id": list(cell_id_to_name.keys()),
                "cell_id": list(cell_id_to_name.values()),
            }
        )
        # Join with mapping DataFrame
        cell_qc_pl = cell_qc_pl.join(cell_map_df, on="cell_integer_id", how="left")
        # Fill any missing values with default format
        cell_qc_pl = cell_qc_pl.with_columns(
            pl.col("cell_id").fill_null(
                pl.col("cell_integer_id")
                .cast(pl.Utf8)
                .map_elements(lambda x: f"cell_{x}", return_dtype=pl.Utf8)
            )
        )
    else:
        # Fallback: use cell_integer_id as cell_id
        cell_qc_pl = cell_qc_pl.with_columns(
            pl.col("cell_integer_id").cast(pl.Utf8).alias("cell_id")
        )

    # Add log1p transformed counts if requested
    if log1p:
        cell_qc_pl = cell_qc_pl.with_columns(
            [
                pl.col("total_counts").log1p().alias("log1p_total_counts"),
                pl.col("n_genes_by_counts")
                .log1p()
                .alias("log1p_n_genes_by_counts"),
            ]
        )

    # Convert to pandas for return compatibility
    cell_qc = cell_qc_pl.to_pandas()

    # Calculate gene-level metrics via SQL using simple aggregation (no JOINs)
    gene_qc_sql = """
    SELECT
        e.gene_integer_id,
        COUNT(DISTINCT e.cell_integer_id) AS n_cells_by_counts,
        SUM(e.value) AS total_counts
    FROM expression e
    GROUP BY e.gene_integer_id
    ORDER BY e.gene_integer_id
    """

    gene_qc = adata.slaf.query(gene_qc_sql)

    # Work with polars DataFrames internally
    gene_qc_pl = gene_qc

    # For scanpy compatibility, we need to ensure all genes are present
    # Use in-memory var if available, otherwise fall back to SQL
    if hasattr(adata.slaf, "var") and adata.slaf.var is not None:
        # Use the materialized var metadata directly
        expected_genes_pl = pl.DataFrame(
            {"gene_integer_id": range(len(adata.slaf.var))}
        )
    else:
        expected_genes_sql = """
        SELECT gene_integer_id
        FROM genes
        ORDER BY gene_integer_id
        """
        expected_genes = adata.slaf.query(expected_genes_sql)
        expected_genes_pl = expected_genes

    # Create a complete gene_qc DataFrame with all expected genes using polars
    gene_qc_complete_pl = expected_genes_pl.join(
        gene_qc_pl, on="gene_integer_id", how="left"
    ).fill_null(0)

    # Ensure proper data types
    gene_qc_complete_pl = gene_qc_complete_pl.with_columns(
        [
            pl.col("n_cells_by_counts").cast(pl.Int64),
            pl.col("total_counts").cast(pl.Float64),
        ]
    )

    # Map gene_integer_id to gene names for scanpy compatibility
    if hasattr(adata.slaf, "var") and adata.slaf.var is not None:
        # Create mapping from gene_integer_id to gene names
        # Use polars DataFrame to get gene names
        var_df = adata.slaf.var
        if "gene_id" in var_df.columns:
            gene_id_to_name = dict(
                zip(
                    var_df["gene_integer_id"].to_list(),
                    var_df["gene_id"].to_list(),
                    strict=False,
                )
            )
        else:
            # Fallback: create gene names from integer IDs
            gene_id_to_name = {i: f"gene_{i}" for i in range(len(var_df))}
        # Use vectorized polars operations for mapping
        # Create mapping DataFrame
        gene_map_df = pl.DataFrame(
            {
                "gene_integer_id": list(gene_id_to_name.keys()),
                "gene_id": list(gene_id_to_name.values()),
            }
        )
        # Join with mapping DataFrame
        gene_qc_complete_pl = gene_qc_complete_pl.join(
            gene_map_df, on="gene_integer_id", how="left"
        )
        # Fill any missing values with default format
        gene_qc_complete_pl = gene_qc_complete_pl.with_columns(
            pl.col("gene_id").fill_null(
                pl.col("gene_integer_id")
                .cast(pl.Utf8)
                .map_elements(lambda x: f"gene_{x}", return_dtype=pl.Utf8)
            )
        )
    else:
        # Fallback: use gene_integer_id as gene_id
        gene_qc_complete_pl = gene_qc_complete_pl.with_columns(
            pl.col("gene_integer_id").cast(pl.Utf8).alias("gene_id")
        )

    # Convert to pandas and set gene_id as index
    gene_qc_complete = gene_qc_complete_pl.to_pandas().set_index("gene_id")

    if log1p:
        gene_qc_complete["log1p_total_counts"] = np.log1p(
            gene_qc_complete["total_counts"]
        )
        gene_qc_complete["log1p_n_cells_by_counts"] = np.log1p(
            gene_qc_complete["n_cells_by_counts"]
        )

    if inplace:
        # Update the metadata tables in SLAF
        # This would require implementing metadata updates in SLAF
        # For now, just update the cached obs/var

        # Update obs
        adata._obs = None  # Clear cache
        for _ in cell_qc.iterrows():
            # Would need to implement metadata updates
            pass

        # Update var
        adata._var = None  # Clear cache
        for _ in gene_qc_complete.iterrows():
            # Would need to implement metadata updates
            pass

        return None
    else:
        return cell_qc, gene_qc_complete

`filter_cells(adata: LazyAnnData, min_counts: int | None = None, min_genes: int | None = None, max_counts: int | None = None, max_genes: int | None = None, inplace: bool = True) -> LazyAnnData | None` `staticmethod` ¶

Filter cells based on quality control metrics using lazy evaluation.

This function filters cells based on their total counts and number of genes using SQL aggregation for memory efficiency. It supports both minimum and maximum thresholds for each metric.

Parameters:

Name	Type	Description	Default
`adata`	`LazyAnnData`	LazyAnnData instance containing the single-cell data.	required
`min_counts`	`int \| None`	Minimum total counts per cell. Cells with fewer counts are filtered.	`None`
`min_genes`	`int \| None`	Minimum number of genes per cell. Cells with fewer genes are filtered.	`None`
`max_counts`	`int \| None`	Maximum total counts per cell. Cells with more counts are filtered.	`None`
`max_genes`	`int \| None`	Maximum number of genes per cell. Cells with more genes are filtered.	`None`
`inplace`	`bool`	Whether to modify the adata object in place. Currently not fully implemented in lazy version - returns None when True.	`True`

Returns:

Type	Description
`LazyAnnData \| None`	LazyAnnData \| None: If inplace=False, returns filtered LazyAnnData. If inplace=True, returns None.

Raises:

Type	Description
`ValueError`	If all cells are filtered out by the criteria.
`RuntimeError`	If the SLAF array is not properly initialized.

Examples:

>>> # Filter cells with basic criteria
>>> slaf_array = SLAFArray("path/to/data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> filtered = LazyPreprocessing.filter_cells(
...     adata, min_counts=100, min_genes=50, inplace=False
... )
>>> print(f"Original cells: {adata.n_obs}")
Original cells: 1000
>>> print(f"Filtered cells: {filtered.n_obs}")
Filtered cells: 850

>>> # Filter with maximum thresholds
>>> filtered = LazyPreprocessing.filter_cells(
...     adata, max_counts=10000, max_genes=5000, inplace=False
... )
>>> print(f"Cells after max filtering: {filtered.n_obs}")
Cells after max filtering: 920

>>> # Error when all cells filtered out
>>> try:
...     LazyPreprocessing.filter_cells(adata, min_counts=1000000)
... except ValueError as e:
...     print(f"Error: {e}")
Error: All cells were filtered out

Source code in slaf/integrations/scanpy.py

@staticmethod
def filter_cells(
    adata: LazyAnnData,
    min_counts: int | None = None,
    min_genes: int | None = None,
    max_counts: int | None = None,
    max_genes: int | None = None,
    inplace: bool = True,
) -> LazyAnnData | None:
    """
    Filter cells based on quality control metrics using lazy evaluation.

    This function filters cells based on their total counts and number of genes
    using SQL aggregation for memory efficiency. It supports both minimum and
    maximum thresholds for each metric.

    Args:
        adata: LazyAnnData instance containing the single-cell data.
        min_counts: Minimum total counts per cell. Cells with fewer counts are filtered.
        min_genes: Minimum number of genes per cell. Cells with fewer genes are filtered.
        max_counts: Maximum total counts per cell. Cells with more counts are filtered.
        max_genes: Maximum number of genes per cell. Cells with more genes are filtered.
        inplace: Whether to modify the adata object in place. Currently not fully
                implemented in lazy version - returns None when True.

    Returns:
        LazyAnnData | None: If inplace=False, returns filtered LazyAnnData.
                           If inplace=True, returns None.

    Raises:
        ValueError: If all cells are filtered out by the criteria.
        RuntimeError: If the SLAF array is not properly initialized.

    Examples:
        >>> # Filter cells with basic criteria
        >>> slaf_array = SLAFArray("path/to/data.slaf")
        >>> adata = LazyAnnData(slaf_array)
        >>> filtered = LazyPreprocessing.filter_cells(
        ...     adata, min_counts=100, min_genes=50, inplace=False
        ... )
        >>> print(f"Original cells: {adata.n_obs}")
        Original cells: 1000
        >>> print(f"Filtered cells: {filtered.n_obs}")
        Filtered cells: 850

        >>> # Filter with maximum thresholds
        >>> filtered = LazyPreprocessing.filter_cells(
        ...     adata, max_counts=10000, max_genes=5000, inplace=False
        ... )
        >>> print(f"Cells after max filtering: {filtered.n_obs}")
        Cells after max filtering: 920

        >>> # Error when all cells filtered out
        >>> try:
        ...     LazyPreprocessing.filter_cells(adata, min_counts=1000000)
        ... except ValueError as e:
        ...     print(f"Error: {e}")
        Error: All cells were filtered out
    """

    # Build filter conditions
    conditions = []

    if min_counts is not None:
        conditions.append(f"cell_stats.total_counts >= {min_counts}")
    if max_counts is not None:
        conditions.append(f"cell_stats.total_counts <= {max_counts}")
    if min_genes is not None:
        conditions.append(f"cell_stats.n_genes_by_counts >= {min_genes}")
    if max_genes is not None:
        conditions.append(f"cell_stats.n_genes_by_counts <= {max_genes}")

    if not conditions:
        return adata if not inplace else None

    where_clause = " AND ".join(conditions)

    # Get filtered cell IDs using simple aggregation (no JOINs)
    filter_sql = f"""
    SELECT cell_stats.cell_integer_id
    FROM (
        SELECT
            e.cell_integer_id,
            COUNT(DISTINCT e.gene_integer_id) as n_genes_by_counts,
            SUM(e.value) as total_counts
        FROM expression e
        GROUP BY e.cell_integer_id
    ) cell_stats
    WHERE ({where_clause})
    ORDER BY cell_stats.cell_integer_id
    """

    filtered_cells = adata.slaf.query(filter_sql)

    if len(filtered_cells) == 0:
        raise ValueError("All cells were filtered out")

    # Create boolean mask from the filtered cell IDs
    # Use the materialized obs metadata to map cell_integer_id to cell names
    if hasattr(adata.slaf, "obs") and adata.slaf.obs is not None:
        # Get all cell integer IDs that pass the filter
        filtered_cell_ids = set(filtered_cells["cell_integer_id"].to_list())

        # Create boolean mask for all cells
        all_cell_ids = adata.slaf.obs["cell_integer_id"].to_list()
        boolean_mask = [cell_id in filtered_cell_ids for cell_id in all_cell_ids]

        # Actually subset the LazyAnnData object (like scanpy does)
        if inplace:
            # Set up the filtered obs function to apply the boolean mask
            def filtered_obs() -> pd.DataFrame:
                obs_df = adata.slaf.obs
                if obs_df is not None:
                    # Work with polars DataFrame internally
                    obs_pl = obs_df
                    # Drop cell_integer_id column if present
                    if "cell_integer_id" in obs_pl.columns:
                        obs_pl = obs_pl.drop("cell_integer_id")
                    # Apply boolean mask
                    obs_pl = obs_pl.filter(pl.Series(boolean_mask))
                    # Convert to pandas DataFrame for AnnData compatibility
                    obs_copy = obs_pl.to_pandas()
                    # Set cell_id as index if present
                    if "cell_id" in obs_copy.columns:
                        obs_copy = obs_copy.set_index("cell_id")
                    # Set index name to match AnnData format
                    if hasattr(obs_copy, "index"):
                        obs_copy.index.name = "cell_id"
                    return obs_copy
                return pd.DataFrame()

            # Set the filtered functions
            adata._filtered_obs = filtered_obs
            adata._filtered_var = None  # No gene filtering
            adata._cell_selector = boolean_mask
            adata._gene_selector = None

            # Clear cached names to force recalculation
            adata._cached_obs_names = None
            adata._cached_var_names = None

            return None
        else:
            # Create a new filtered LazyAnnData
            filtered_adata = LazyAnnData(adata.slaf, backend=adata.backend)

            # Set up the filtered obs function
            def filtered_obs() -> pd.DataFrame:
                obs_df = adata.slaf.obs
                if obs_df is not None:
                    # Work with polars DataFrame internally
                    obs_pl = obs_df
                    # Drop cell_integer_id column if present
                    if "cell_integer_id" in obs_pl.columns:
                        obs_pl = obs_pl.drop("cell_integer_id")
                    # Apply boolean mask
                    obs_pl = obs_pl.filter(pl.Series(boolean_mask))
                    # Convert to pandas DataFrame for AnnData compatibility
                    obs_copy = obs_pl.to_pandas()
                    # Set cell_id as index if present
                    if "cell_id" in obs_copy.columns:
                        obs_copy = obs_copy.set_index("cell_id")
                    # Set index name to match AnnData format
                    if hasattr(obs_copy, "index"):
                        obs_copy.index.name = "cell_id"
                    return obs_copy
                return pd.DataFrame()

            # Set the filtered functions
            filtered_adata._filtered_obs = filtered_obs
            filtered_adata._filtered_var = None  # No gene filtering
            filtered_adata._cell_selector = boolean_mask
            filtered_adata._gene_selector = None

            return filtered_adata
    else:
        # Fallback: create boolean mask by mapping integer IDs to cell names
        # obs_names contains string cell IDs, so we need to map integer IDs to cell names
        if hasattr(adata.slaf, "obs") and adata.slaf.obs is not None:
            # Create mapping from cell_integer_id to cell_id
            obs_df = adata.slaf.obs
            if "cell_id" in obs_df.columns and "cell_integer_id" in obs_df.columns:
                id_to_name = dict(
                    zip(
                        obs_df["cell_integer_id"].to_list(),
                        obs_df["cell_id"].to_list(),
                        strict=False,
                    )
                )
                # Map filtered integer IDs to cell names
                filtered_cell_names = {
                    id_to_name.get(cid)
                    for cid in filtered_cells["cell_integer_id"].to_list()
                    if id_to_name.get(cid) is not None
                }
                # Create boolean mask
                cell_mask = adata.obs_names.isin(filtered_cell_names)
            else:
                # Fallback: use integer IDs directly (assume obs_names are integer strings)
                filtered_cell_ids = {
                    str(cid) for cid in filtered_cells["cell_integer_id"].to_list()
                }
                cell_mask = adata.obs_names.astype(str).isin(filtered_cell_ids)
        else:
            # Last resort: assume obs_names match integer IDs as strings
            filtered_cell_ids = {
                str(cid) for cid in filtered_cells["cell_integer_id"].to_list()
            }
            cell_mask = adata.obs_names.astype(str).isin(filtered_cell_ids)

    if inplace:
        # Apply filter to adata (would need proper implementation)
        # For now, just return the original adata
        print(
            f"Filtered out {np.sum(~cell_mask)} cells, {np.sum(cell_mask)} remaining"
        )
        return None
    else:
        # Create new filtered LazyAnnData
        filtered_adata = adata.copy()
        # Apply mask (would need proper implementation)
        return filtered_adata

`filter_genes(adata: LazyAnnData, min_counts: int | None = None, min_cells: int | None = None, max_counts: int | None = None, max_cells: int | None = None, inplace: bool = True) -> LazyAnnData | None` `staticmethod` ¶

Filter genes based on quality control metrics using lazy evaluation.

This function filters genes based on their total counts and number of cells using SQL aggregation for memory efficiency. It supports both minimum and maximum thresholds for each metric.

Parameters:

Name	Type	Description	Default
`adata`	`LazyAnnData`	LazyAnnData instance containing the single-cell data.	required
`min_counts`	`int \| None`	Minimum total counts per gene. Genes with fewer counts are filtered.	`None`
`min_cells`	`int \| None`	Minimum number of cells per gene. Genes expressed in fewer cells are filtered.	`None`
`max_counts`	`int \| None`	Maximum total counts per gene. Genes with more counts are filtered.	`None`
`max_cells`	`int \| None`	Maximum number of cells per gene. Genes expressed in more cells are filtered.	`None`
`inplace`	`bool`	Whether to modify the adata object in place. Currently not fully implemented in lazy version - returns None when True.	`True`

Returns:

Type	Description
`LazyAnnData \| None`	LazyAnnData \| None: If inplace=False, returns filtered LazyAnnData. If inplace=True, returns None.

Raises:

Type	Description
`ValueError`	If all genes are filtered out by the criteria.
`RuntimeError`	If the SLAF array is not properly initialized.

Examples:

>>> # Filter genes with basic criteria
>>> slaf_array = SLAFArray("path/to/data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> filtered = LazyPreprocessing.filter_genes(
...     adata, min_counts=10, min_cells=5, inplace=False
... )
>>> print(f"Original genes: {adata.n_vars}")
Original genes: 20000
>>> print(f"Filtered genes: {filtered.n_vars}")
Filtered genes: 15000

>>> # Filter with maximum thresholds
>>> filtered = LazyPreprocessing.filter_genes(
...     adata, max_counts=100000, max_cells=1000, inplace=False
... )
>>> print(f"Genes after max filtering: {filtered.n_vars}")
Genes after max filtering: 18000

>>> # Error when all genes filtered out
>>> try:
...     LazyPreprocessing.filter_genes(adata, min_counts=1000000)
... except ValueError as e:
...     print(f"Error: {e}")
Error: All genes were filtered out

Source code in slaf/integrations/scanpy.py

@staticmethod
def filter_genes(
    adata: LazyAnnData,
    min_counts: int | None = None,
    min_cells: int | None = None,
    max_counts: int | None = None,
    max_cells: int | None = None,
    inplace: bool = True,
) -> LazyAnnData | None:
    """
    Filter genes based on quality control metrics using lazy evaluation.

    This function filters genes based on their total counts and number of cells
    using SQL aggregation for memory efficiency. It supports both minimum and
    maximum thresholds for each metric.

    Args:
        adata: LazyAnnData instance containing the single-cell data.
        min_counts: Minimum total counts per gene. Genes with fewer counts are filtered.
        min_cells: Minimum number of cells per gene. Genes expressed in fewer cells are filtered.
        max_counts: Maximum total counts per gene. Genes with more counts are filtered.
        max_cells: Maximum number of cells per gene. Genes expressed in more cells are filtered.
        inplace: Whether to modify the adata object in place. Currently not fully
                implemented in lazy version - returns None when True.

    Returns:
        LazyAnnData | None: If inplace=False, returns filtered LazyAnnData.
                           If inplace=True, returns None.

    Raises:
        ValueError: If all genes are filtered out by the criteria.
        RuntimeError: If the SLAF array is not properly initialized.

    Examples:
        >>> # Filter genes with basic criteria
        >>> slaf_array = SLAFArray("path/to/data.slaf")
        >>> adata = LazyAnnData(slaf_array)
        >>> filtered = LazyPreprocessing.filter_genes(
        ...     adata, min_counts=10, min_cells=5, inplace=False
        ... )
        >>> print(f"Original genes: {adata.n_vars}")
        Original genes: 20000
        >>> print(f"Filtered genes: {filtered.n_vars}")
        Filtered genes: 15000

        >>> # Filter with maximum thresholds
        >>> filtered = LazyPreprocessing.filter_genes(
        ...     adata, max_counts=100000, max_cells=1000, inplace=False
        ... )
        >>> print(f"Genes after max filtering: {filtered.n_vars}")
        Genes after max filtering: 18000

        >>> # Error when all genes filtered out
        >>> try:
        ...     LazyPreprocessing.filter_genes(adata, min_counts=1000000)
        ... except ValueError as e:
        ...     print(f"Error: {e}")
        Error: All genes were filtered out
    """

    # Build filter conditions for genes
    conditions = []

    if min_counts is not None:
        conditions.append(f"gene_stats.total_counts >= {min_counts}")
    if max_counts is not None:
        conditions.append(f"gene_stats.total_counts <= {max_counts}")
    if min_cells is not None:
        conditions.append(f"gene_stats.n_cells_by_counts >= {min_cells}")
    if max_cells is not None:
        conditions.append(f"gene_stats.n_cells_by_counts <= {max_cells}")

    if not conditions:
        return adata if not inplace else None

    where_clause = " AND ".join(conditions)

    # Get filtered gene IDs using simple aggregation (no JOINs)
    filter_sql = f"""
    SELECT gene_stats.gene_integer_id
    FROM (
        SELECT
            e.gene_integer_id,
            COUNT(DISTINCT e.cell_integer_id) AS n_cells_by_counts,
            SUM(e.value) AS total_counts
        FROM expression e
        GROUP BY e.gene_integer_id
    ) gene_stats
    WHERE {where_clause}
    ORDER BY gene_stats.gene_integer_id
    """

    filtered_genes = adata.slaf.query(filter_sql)

    if len(filtered_genes) == 0:
        raise ValueError("All genes were filtered out")

    # Create boolean mask from the filtered gene IDs
    # Use the materialized var metadata to map gene_integer_id to gene names
    if hasattr(adata.slaf, "var") and adata.slaf.var is not None:
        # Create a mapping from gene_integer_id to gene names
        # Use polars DataFrame to get gene names
        var_df = adata.slaf.var
        if "gene_id" in var_df.columns:
            gene_id_to_name = dict(
                zip(
                    var_df["gene_integer_id"].to_list(),
                    var_df["gene_id"].to_list(),
                    strict=False,
                )
            )
        else:
            # Fallback: create gene names from integer IDs
            gene_id_to_name = {i: f"gene_{i}" for i in range(len(var_df))}
        filtered_gene_names = [
            gene_id_to_name.get(gid, f"gene_{gid}")
            for gid in filtered_genes["gene_integer_id"]
        ]
        gene_mask = adata.var_names.isin(filtered_gene_names)
    else:
        # Fallback to using gene_integer_id directly
        gene_mask = adata.var_names.isin(filtered_genes["gene_integer_id"])

    if inplace:
        # Apply filter to adata (would need proper implementation)
        print(
            f"Filtered out {np.sum(~gene_mask)} genes, {np.sum(gene_mask)} remaining"
        )
        return None
    else:
        # Create new filtered LazyAnnData
        filtered_adata = adata.copy()
        # Apply mask (would need proper implementation)
        return filtered_adata

`normalize_total(adata: LazyAnnData, target_sum: float | None = 10000.0, inplace: bool = True, fragments: bool | None = None) -> LazyAnnData | None` `staticmethod` ¶

Normalize counts per cell to target sum using lazy evaluation.

This function normalizes the expression data so that each cell has a total count equal to target_sum. The normalization is applied lazily and stored as a transformation that will be computed when the data is accessed.

Parameters:

Name	Type	Description	Default
`adata`	`LazyAnnData`	LazyAnnData instance containing the single-cell data.	required
`target_sum`	`float \| None`	Target sum for normalization. Default is 10,000.	`10000.0`
`inplace`	`bool`	Whether to modify the adata object in place. If False, returns a copy with the transformation applied.	`True`
`fragments`	`bool \| None`	Whether to use fragment-based processing. If None, automatically selects based on dataset characteristics.	`None`

Returns:

Type	Description
`LazyAnnData \| None`	LazyAnnData \| None: If inplace=False, returns LazyAnnData with transformation. If inplace=True, returns None.

Raises:

Type	Description
`ValueError`	If target_sum is not positive.
`RuntimeError`	If the SLAF array is not properly initialized.

Examples:

>>> # Basic normalization
>>> slaf_array = SLAFArray("path/to/data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> LazyPreprocessing.normalize_total(adata, target_sum=10000)
Applied normalize_total with target_sum=10000

>>> # Custom target sum
>>> adata_copy = adata.copy()
>>> LazyPreprocessing.normalize_total(
...     adata_copy, target_sum=5000, inplace=False
... )
>>> print("Normalization applied to copy")

>>> # Error with invalid target_sum
>>> try:
...     LazyPreprocessing.normalize_total(adata, target_sum=0)
... except ValueError as e:
...     print(f"Error: {e}")
Error: target_sum must be positive

Source code in slaf/integrations/scanpy.py

@staticmethod
def normalize_total(
    adata: LazyAnnData,
    target_sum: float | None = 1e4,
    inplace: bool = True,
    fragments: bool | None = None,
) -> LazyAnnData | None:
    """
    Normalize counts per cell to target sum using lazy evaluation.

    This function normalizes the expression data so that each cell has a total
    count equal to target_sum. The normalization is applied lazily and stored
    as a transformation that will be computed when the data is accessed.

    Args:
        adata: LazyAnnData instance containing the single-cell data.
        target_sum: Target sum for normalization. Default is 10,000.
        inplace: Whether to modify the adata object in place. If False, returns
                a copy with the transformation applied.
        fragments: Whether to use fragment-based processing. If None, automatically
                  selects based on dataset characteristics.

    Returns:
        LazyAnnData | None: If inplace=False, returns LazyAnnData with transformation.
                           If inplace=True, returns None.

    Raises:
        ValueError: If target_sum is not positive.
        RuntimeError: If the SLAF array is not properly initialized.

    Examples:
        >>> # Basic normalization
        >>> slaf_array = SLAFArray("path/to/data.slaf")
        >>> adata = LazyAnnData(slaf_array)
        >>> LazyPreprocessing.normalize_total(adata, target_sum=10000)
        Applied normalize_total with target_sum=10000

        >>> # Custom target sum
        >>> adata_copy = adata.copy()
        >>> LazyPreprocessing.normalize_total(
        ...     adata_copy, target_sum=5000, inplace=False
        ... )
        >>> print("Normalization applied to copy")

        >>> # Error with invalid target_sum
        >>> try:
        ...     LazyPreprocessing.normalize_total(adata, target_sum=0)
        ... except ValueError as e:
        ...     print(f"Error: {e}")
        Error: target_sum must be positive
    """
    if target_sum is None:
        target_sum = 1e4

    # Validate target_sum
    if target_sum <= 0:
        raise ValueError("target_sum must be positive")

    # Determine processing strategy
    if fragments is not None:
        use_fragments = fragments
    else:
        # Check if dataset has multiple fragments
        try:
            fragments_list = adata.slaf.expression.get_fragments()
            use_fragments = len(fragments_list) > 1
        except Exception:
            use_fragments = False

    if use_fragments:
        # Use fragment-based processing
        try:
            from slaf.core.fragment_processor import FragmentProcessor

            # Get selectors from the LazyAnnData if it's sliced
            cell_selector = getattr(adata, "_cell_selector", None)
            gene_selector = getattr(adata, "_gene_selector", None)

            processor = FragmentProcessor(
                adata.slaf,
                cell_selector=cell_selector,
                gene_selector=gene_selector,
                max_workers=4,
                enable_caching=True,
            )
            # Use smart strategy selection for optimal performance
            lazy_pipeline = processor.build_lazy_pipeline_smart(
                "normalize_total", target_sum=target_sum
            )
            result_df = processor.compute(lazy_pipeline)

            # Update adata with normalized values
            return adata._update_with_normalized_data(
                result_df, target_sum, inplace
            )

        except Exception as e:
            print(
                f"Fragment processing failed, falling back to global processing: {e}"
            )
            # Fall back to global processing
            use_fragments = False

    if not use_fragments:
        # Use global processing (original implementation)
        # Get cell totals for normalization using only the expression table
        cell_totals_sql = """
        SELECT
            e.cell_integer_id,
            SUM(e.value) as total_counts
        FROM expression e
        GROUP BY e.cell_integer_id
        ORDER BY e.cell_integer_id
        """

        cell_totals = adata.slaf.query(cell_totals_sql)

    # Work with polars DataFrame internally
    cell_totals_pl = cell_totals

    # Create normalization factors using polars
    # Map cell_integer_id to cell names for compatibility with anndata.py
    if hasattr(adata.slaf, "obs") and adata.slaf.obs is not None:
        # Create mapping from cell_integer_id to cell names
        # Use polars DataFrame to get cell names
        obs_df = adata.slaf.obs
        if "cell_id" in obs_df.columns:
            cell_id_to_name = dict(
                zip(
                    obs_df["cell_integer_id"].to_list(),
                    obs_df["cell_id"].to_list(),
                    strict=False,
                )
            )
        else:
            # Fallback: create cell names from integer IDs
            cell_id_to_name = {i: f"cell_{i}" for i in range(len(obs_df))}
        # Use vectorized polars operations for mapping
        # Create mapping DataFrame
        cell_map_df = pl.DataFrame(
            {
                "cell_integer_id": list(cell_id_to_name.keys()),
                "cell_id": list(cell_id_to_name.values()),
            }
        )
        # Join with mapping DataFrame
        cell_totals_pl = cell_totals_pl.join(
            cell_map_df, on="cell_integer_id", how="left"
        )
        # Fill any missing values with default format
        cell_totals_pl = cell_totals_pl.with_columns(
            [
                pl.col("cell_id").fill_null(
                    pl.col("cell_integer_id")
                    .cast(pl.Utf8)
                    .map_elements(lambda x: f"cell_{x}", return_dtype=pl.Utf8)
                ),
                (target_sum / pl.col("total_counts")).alias("normalization_factor"),
            ]
        )
        # Convert to dictionary for compatibility
        normalization_dict = dict(
            zip(
                cell_totals_pl["cell_id"].to_list(),
                cell_totals_pl["normalization_factor"].to_list(),
                strict=False,
            )
        )
    else:
        # Fallback: use cell_integer_id as string keys
        cell_totals_pl = cell_totals_pl.with_columns(
            [
                pl.col("cell_integer_id").cast(pl.Utf8).alias("cell_id"),
                (target_sum / pl.col("total_counts")).alias("normalization_factor"),
            ]
        )
        # Convert to dictionary for compatibility
        normalization_dict = dict(
            zip(
                cell_totals_pl["cell_id"].to_list(),
                cell_totals_pl["normalization_factor"].to_list(),
                strict=False,
            )
        )

    if inplace:
        # Store normalization factors for lazy application
        if not hasattr(adata, "_transformations"):
            adata._transformations = {}

        adata._transformations["normalize_total"] = {
            "type": "normalize_total",
            "target_sum": float(
                f"{target_sum:.10f}"
            ),  # Convert to regular decimal format
            "cell_factors": normalization_dict,
        }

        print(f"Applied normalize_total with target_sum={target_sum}")
        return None
    else:
        # Create a copy with the transformation (copy-on-write)
        new_adata = adata.copy()
        if not hasattr(new_adata, "_transformations"):
            new_adata._transformations = {}

        new_adata._transformations["normalize_total"] = {
            "type": "normalize_total",
            "target_sum": float(
                f"{target_sum:.10f}"
            ),  # Convert to regular decimal format
            "cell_factors": normalization_dict,
        }

        return new_adata

`log1p(adata: LazyAnnData, inplace: bool = True, fragments: bool | None = None) -> LazyAnnData | None` `staticmethod` ¶

Apply log1p transformation to expression data using lazy evaluation.

This function applies log(1 + x) transformation to the expression data. The transformation is stored lazily and will be computed when the data is accessed, avoiding memory-intensive operations on large datasets.

Parameters:

Name	Type	Description	Default
`adata`	`LazyAnnData`	LazyAnnData instance containing the single-cell data.	required
`inplace`	`bool`	Whether to modify the adata object in place. If False, returns a copy with the transformation applied.	`True`

Returns:

Type	Description
`LazyAnnData \| None`	LazyAnnData \| None: If inplace=False, returns LazyAnnData with transformation. If inplace=True, returns None.

Raises:

Type	Description
`RuntimeError`	If the SLAF array is not properly initialized.

Examples:

>>> # Apply log1p transformation
>>> slaf_array = SLAFArray("path/to/data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> LazyPreprocessing.log1p(adata)
Applied log1p transformation

>>> # Apply to copy
>>> adata_copy = adata.copy()
>>> LazyPreprocessing.log1p(adata_copy, inplace=False)
>>> print("Log1p transformation applied to copy")

>>> # Check transformation was stored
>>> print("log1p" in adata._transformations)
True

Source code in slaf/integrations/scanpy.py

@staticmethod
def log1p(
    adata: LazyAnnData, inplace: bool = True, fragments: bool | None = None
) -> LazyAnnData | None:
    """
    Apply log1p transformation to expression data using lazy evaluation.

    This function applies log(1 + x) transformation to the expression data.
    The transformation is stored lazily and will be computed when the data
    is accessed, avoiding memory-intensive operations on large datasets.

    Args:
        adata: LazyAnnData instance containing the single-cell data.
        inplace: Whether to modify the adata object in place. If False, returns
                a copy with the transformation applied.

    Returns:
        LazyAnnData | None: If inplace=False, returns LazyAnnData with transformation.
                           If inplace=True, returns None.

    Raises:
        RuntimeError: If the SLAF array is not properly initialized.

    Examples:
        >>> # Apply log1p transformation
        >>> slaf_array = SLAFArray("path/to/data.slaf")
        >>> adata = LazyAnnData(slaf_array)
        >>> LazyPreprocessing.log1p(adata)
        Applied log1p transformation

        >>> # Apply to copy
        >>> adata_copy = adata.copy()
        >>> LazyPreprocessing.log1p(adata_copy, inplace=False)
        >>> print("Log1p transformation applied to copy")

        >>> # Check transformation was stored
        >>> print("log1p" in adata._transformations)
        True
    """
    # Determine processing strategy
    if fragments is not None:
        use_fragments = fragments
    else:
        # Check if dataset has multiple fragments
        try:
            fragments_list = adata.slaf.expression.get_fragments()
            use_fragments = len(fragments_list) > 1
        except Exception:
            use_fragments = False

    if use_fragments:
        # Use fragment-based processing
        try:
            from slaf.core.fragment_processor import FragmentProcessor

            # Get selectors from the LazyAnnData if it's sliced
            cell_selector = getattr(adata, "_cell_selector", None)
            gene_selector = getattr(adata, "_gene_selector", None)

            processor = FragmentProcessor(
                adata.slaf,
                cell_selector=cell_selector,
                gene_selector=gene_selector,
                max_workers=4,
                enable_caching=True,
            )
            # Use smart strategy selection for optimal performance
            lazy_pipeline = processor.build_lazy_pipeline_smart("log1p")
            result_df = processor.compute(lazy_pipeline)

            # Update adata with log1p values
            return adata._update_with_log1p_data(result_df, inplace)

        except Exception as e:
            print(
                f"Fragment processing failed, falling back to global processing: {e}"
            )
            # Fall back to global processing
            use_fragments = False

    if not use_fragments:
        # Use global processing (original implementation)
        if inplace:
            # Store log1p transformation for lazy application
            if not hasattr(adata, "_transformations"):
                adata._transformations = {}

            adata._transformations["log1p"] = {"type": "log1p", "applied": True}

            print("Applied log1p transformation")
            return None
        else:
            # Create a copy with the transformation (copy-on-write)
            new_adata = adata.copy()
            if not hasattr(new_adata, "_transformations"):
                new_adata._transformations = {}

            new_adata._transformations["log1p"] = {"type": "log1p", "applied": True}

            return new_adata

`highly_variable_genes(adata: LazyAnnData, min_mean: float = 0.0125, max_mean: float = 3, min_disp: float = 0.5, max_disp: float = np.inf, n_top_genes: int | None = None, inplace: bool = True) -> pd.DataFrame | None` `staticmethod` ¶

Identify highly variable genes using lazy evaluation.

This function identifies genes with high cell-to-cell variation in expression using SQL aggregation for memory efficiency. It calculates mean expression and dispersion for each gene and applies filtering criteria.

Parameters:

Name	Type	Description	Default
`adata`	`LazyAnnData`	LazyAnnData instance containing the single-cell data.	required
`min_mean`	`float`	Minimum mean expression for genes to be considered.	`0.0125`
`max_mean`	`float`	Maximum mean expression for genes to be considered.	`3`
`min_disp`	`float`	Minimum dispersion for genes to be considered.	`0.5`
`max_disp`	`float`	Maximum dispersion for genes to be considered.	`inf`
`n_top_genes`	`int \| None`	Number of top genes to select by dispersion. If specified, overrides min_disp and max_disp criteria.	`None`
`inplace`	`bool`	Whether to modify the adata object in place. Currently not fully implemented - returns None when True.	`True`

Returns:

Type	Description
`DataFrame \| None`	pd.DataFrame \| None: If inplace=False, returns DataFrame with gene statistics and highly_variable column. If inplace=True, returns None.

Raises:

Type	Description
`RuntimeError`	If the SLAF array is not properly initialized.

Examples:

>>> # Find highly variable genes
>>> slaf_array = SLAFArray("path/to/data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> hvg_stats = LazyPreprocessing.highly_variable_genes(
...     adata, inplace=False
... )
>>> print(f"Highly variable genes: {hvg_stats['highly_variable'].sum()}")
Highly variable genes: 1500

>>> # With custom criteria
>>> hvg_stats = LazyPreprocessing.highly_variable_genes(
...     adata, min_mean=0.1, max_mean=5, min_disp=1.0, inplace=False
... )
>>> print(f"Genes meeting criteria: {hvg_stats['highly_variable'].sum()}")
Genes meeting criteria: 800

>>> # Select top N genes
>>> hvg_stats = LazyPreprocessing.highly_variable_genes(
...     adata, n_top_genes=1000, inplace=False
... )
>>> print(f"Top genes selected: {hvg_stats['highly_variable'].sum()}")
Top genes selected: 1000

Source code in slaf/integrations/scanpy.py

@staticmethod
def highly_variable_genes(
    adata: LazyAnnData,
    min_mean: float = 0.0125,
    max_mean: float = 3,
    min_disp: float = 0.5,
    max_disp: float = np.inf,
    n_top_genes: int | None = None,
    inplace: bool = True,
) -> pd.DataFrame | None:
    """
    Identify highly variable genes using lazy evaluation.

    This function identifies genes with high cell-to-cell variation in expression
    using SQL aggregation for memory efficiency. It calculates mean expression
    and dispersion for each gene and applies filtering criteria.

    Args:
        adata: LazyAnnData instance containing the single-cell data.
        min_mean: Minimum mean expression for genes to be considered.
        max_mean: Maximum mean expression for genes to be considered.
        min_disp: Minimum dispersion for genes to be considered.
        max_disp: Maximum dispersion for genes to be considered.
        n_top_genes: Number of top genes to select by dispersion.
                    If specified, overrides min_disp and max_disp criteria.
        inplace: Whether to modify the adata object in place. Currently not
                fully implemented - returns None when True.

    Returns:
        pd.DataFrame | None: If inplace=False, returns DataFrame with gene statistics
                            and highly_variable column. If inplace=True, returns None.

    Raises:
        RuntimeError: If the SLAF array is not properly initialized.

    Examples:
        >>> # Find highly variable genes
        >>> slaf_array = SLAFArray("path/to/data.slaf")
        >>> adata = LazyAnnData(slaf_array)
        >>> hvg_stats = LazyPreprocessing.highly_variable_genes(
        ...     adata, inplace=False
        ... )
        >>> print(f"Highly variable genes: {hvg_stats['highly_variable'].sum()}")
        Highly variable genes: 1500

        >>> # With custom criteria
        >>> hvg_stats = LazyPreprocessing.highly_variable_genes(
        ...     adata, min_mean=0.1, max_mean=5, min_disp=1.0, inplace=False
        ... )
        >>> print(f"Genes meeting criteria: {hvg_stats['highly_variable'].sum()}")
        Genes meeting criteria: 800

        >>> # Select top N genes
        >>> hvg_stats = LazyPreprocessing.highly_variable_genes(
        ...     adata, n_top_genes=1000, inplace=False
        ... )
        >>> print(f"Top genes selected: {hvg_stats['highly_variable'].sum()}")
        Top genes selected: 1000
    """

    # Calculate gene statistics via SQL using simple aggregation (no JOINs)
    stats_sql = """
    SELECT
        e.gene_integer_id,
        COUNT(DISTINCT e.cell_integer_id) AS n_cells,
        AVG(e.value) AS mean_expr,
        VARIANCE(e.value) AS variance,
        CASE WHEN AVG(e.value) > 0 THEN VARIANCE(e.value) / AVG(e.value) ELSE 0 END as dispersion
    FROM expression e
    GROUP BY e.gene_integer_id
    ORDER BY e.gene_integer_id
    """

    gene_stats = adata.slaf.query(stats_sql)

    # Work with polars DataFrames internally
    gene_stats_pl = gene_stats

    # Get the expected gene_integer_ids from the materialized var metadata
    # Use in-memory var if available, otherwise fall back to SQL
    if hasattr(adata.slaf, "var") and adata.slaf.var is not None:
        # Use the materialized var metadata directly
        expected_genes_pl = pl.DataFrame(
            {"gene_integer_id": range(len(adata.slaf.var))}
        )
    else:
        # Fallback to SQL if var is not available
        expected_genes_sql = """
        SELECT gene_integer_id
        FROM genes
        ORDER BY gene_integer_id
        """
        expected_genes = adata.slaf.query(expected_genes_sql)
        expected_genes_pl = expected_genes

    # Create a complete gene_stats DataFrame with all expected genes using polars
    gene_stats_complete_pl = expected_genes_pl.join(
        gene_stats_pl, on="gene_integer_id", how="left"
    ).fill_null(0)

    # Ensure proper data types
    gene_stats_complete_pl = gene_stats_complete_pl.with_columns(
        [
            pl.col("n_cells").cast(pl.Int64),
            pl.col("mean_expr").cast(pl.Float64),
            pl.col("variance").cast(pl.Float64),
            pl.col("dispersion").cast(pl.Float64),
        ]
    )

    # Map gene_integer_id to gene_id for scanpy compatibility
    if hasattr(adata.slaf, "var") and adata.slaf.var is not None:
        # Create mapping from gene_integer_id to gene names
        # Use polars DataFrame to get gene names
        var_df = adata.slaf.var
        if "gene_id" in var_df.columns:
            gene_id_to_name = dict(
                zip(
                    var_df["gene_integer_id"].to_list(),
                    var_df["gene_id"].to_list(),
                    strict=False,
                )
            )
        else:
            # Fallback: create gene names from integer IDs
            gene_id_to_name = {i: f"gene_{i}" for i in range(len(var_df))}
        # Use vectorized polars operations for mapping
        # Create mapping DataFrame
        gene_map_df = pl.DataFrame(
            {
                "gene_integer_id": list(gene_id_to_name.keys()),
                "gene_id": list(gene_id_to_name.values()),
            }
        )
        # Join with mapping DataFrame
        gene_stats_complete_pl = gene_stats_complete_pl.join(
            gene_map_df, on="gene_integer_id", how="left"
        )
        # Fill any missing values with default format
        gene_stats_complete_pl = gene_stats_complete_pl.with_columns(
            pl.col("gene_id").fill_null(
                pl.col("gene_integer_id")
                .cast(pl.Utf8)
                .map_elements(lambda x: f"gene_{x}", return_dtype=pl.Utf8)
            )
        )
    else:
        # Fallback: use gene_integer_id as gene_id
        gene_stats_complete_pl = gene_stats_complete_pl.with_columns(
            pl.col("gene_integer_id").cast(pl.Utf8).alias("gene_id")
        )

    # Convert to pandas and set gene_id as index for scanpy compatibility
    gene_stats_complete = gene_stats_complete_pl.to_pandas().set_index("gene_id")

    # Apply HVG criteria
    hvg_mask = (
        (gene_stats_complete["mean_expr"] >= min_mean)
        & (gene_stats_complete["mean_expr"] <= max_mean)
        & (gene_stats_complete["dispersion"] >= min_disp)
        & (gene_stats_complete["dispersion"] <= max_disp)
    )

    gene_stats_complete["highly_variable"] = hvg_mask

    if n_top_genes is not None:
        # Select top N genes by dispersion
        top_genes = gene_stats_complete.nlargest(n_top_genes, "dispersion")
        gene_stats_complete["highly_variable"] = gene_stats_complete.index.isin(
            top_genes.index
        )

    if inplace:
        # Update var metadata (would need implementation)
        print(f"Identified {hvg_mask.sum()} highly variable genes")
        return None
    else:
        return gene_stats_complete

Functions¶

`apply_transformations(adata: LazyAnnData, transformations: list | None = None) -> LazyAnnData` ¶

Apply a list of transformations to the data.

This function provides explicit control over when transformations are applied. It creates a copy of the LazyAnnData with only the specified transformations stored for lazy evaluation.

Parameters:

Name	Type	Description	Default
`adata`	`LazyAnnData`	LazyAnnData instance containing the single-cell data.	required
`transformations`	`list \| None`	List of transformation names to apply. If None, applies all available transformations.	`None`

Returns:

Name	Type	Description
`LazyAnnData`	`LazyAnnData`	New LazyAnnData with specified transformations applied.

Raises:

Type	Description
`RuntimeError`	If the SLAF array is not properly initialized.

Examples:

>>> # Apply all transformations
>>> slaf_array = SLAFArray("path/to/data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> LazyPreprocessing.normalize_total(adata)
>>> LazyPreprocessing.log1p(adata)
>>> adata_with_transforms = apply_transformations(adata)
>>> print("Transformations applied")

>>> # Apply specific transformations
>>> adata_copy = adata.copy()
>>> adata_with_norm = apply_transformations(
...     adata_copy, transformations=["normalize_total"]
... )
>>> print("Only normalization applied")

Source code in slaf/integrations/scanpy.py

def apply_transformations(
    adata: LazyAnnData, transformations: list | None = None
) -> LazyAnnData:
    """
    Apply a list of transformations to the data.

    This function provides explicit control over when transformations are applied.
    It creates a copy of the LazyAnnData with only the specified transformations
    stored for lazy evaluation.

    Args:
        adata: LazyAnnData instance containing the single-cell data.
        transformations: List of transformation names to apply. If None, applies all
                       available transformations.

    Returns:
        LazyAnnData: New LazyAnnData with specified transformations applied.

    Raises:
        RuntimeError: If the SLAF array is not properly initialized.

    Examples:
        >>> # Apply all transformations
        >>> slaf_array = SLAFArray("path/to/data.slaf")
        >>> adata = LazyAnnData(slaf_array)
        >>> LazyPreprocessing.normalize_total(adata)
        >>> LazyPreprocessing.log1p(adata)
        >>> adata_with_transforms = apply_transformations(adata)
        >>> print("Transformations applied")

        >>> # Apply specific transformations
        >>> adata_copy = adata.copy()
        >>> adata_with_norm = apply_transformations(
        ...     adata_copy, transformations=["normalize_total"]
        ... )
        >>> print("Only normalization applied")
    """
    if transformations is None:
        # Apply all transformations
        transformations = list(adata._transformations.keys())

    # Create a copy to avoid modifying the original
    new_adata = adata.copy()

    # Apply only the specified transformations
    new_adata._transformations = {
        name: adata._transformations[name]
        for name in transformations
        if name in adata._transformations
    }

    return new_adata

`clear_transformations(adata: LazyAnnData, inplace: bool = False) -> LazyAnnData | None` ¶

Clear all transformations from the data.

This function removes all stored transformations from the LazyAnnData object, effectively resetting it to its original state without any preprocessing.

Parameters:

Name	Type	Description	Default
`adata`	`LazyAnnData`	LazyAnnData instance containing the single-cell data.	required
`inplace`	`bool`	Whether to modify the adata object in place. If False, returns a copy with transformations cleared.	`False`

Returns:

Type	Description
`LazyAnnData \| None`	LazyAnnData \| None: If inplace=False, returns LazyAnnData with transformations cleared. If inplace=True, returns None.

Raises:

Type	Description
`RuntimeError`	If the SLAF array is not properly initialized.

Examples:

>>> # Clear transformations in place
>>> slaf_array = SLAFArray("path/to/data.slaf")
>>> adata = LazyAnnData(slaf_array)
>>> LazyPreprocessing.normalize_total(adata)
>>> LazyPreprocessing.log1p(adata)
>>> clear_transformations(adata, inplace=True)
>>> print("Transformations cleared")

>>> # Clear transformations to copy
>>> adata_copy = adata.copy()
>>> LazyPreprocessing.normalize_total(adata_copy)
>>> adata_clean = clear_transformations(adata_copy, inplace=False)
>>> print("Clean copy created")

Source code in slaf/integrations/scanpy.py

def clear_transformations(
    adata: LazyAnnData, inplace: bool = False
) -> LazyAnnData | None:
    """
    Clear all transformations from the data.

    This function removes all stored transformations from the LazyAnnData object,
    effectively resetting it to its original state without any preprocessing.

    Args:
        adata: LazyAnnData instance containing the single-cell data.
        inplace: Whether to modify the adata object in place. If False, returns
                a copy with transformations cleared.

    Returns:
        LazyAnnData | None: If inplace=False, returns LazyAnnData with transformations
                           cleared. If inplace=True, returns None.

    Raises:
        RuntimeError: If the SLAF array is not properly initialized.

    Examples:
        >>> # Clear transformations in place
        >>> slaf_array = SLAFArray("path/to/data.slaf")
        >>> adata = LazyAnnData(slaf_array)
        >>> LazyPreprocessing.normalize_total(adata)
        >>> LazyPreprocessing.log1p(adata)
        >>> clear_transformations(adata, inplace=True)
        >>> print("Transformations cleared")

        >>> # Clear transformations to copy
        >>> adata_copy = adata.copy()
        >>> LazyPreprocessing.normalize_total(adata_copy)
        >>> adata_clean = clear_transformations(adata_copy, inplace=False)
        >>> print("Clean copy created")
    """
    if inplace:
        adata._transformations = {}
        return None
    else:
        new_adata = adata.copy()
        new_adata._transformations = {}
        return new_adata

Integrations API¶

AnnData Integration¶

slaf.integrations.anndata ¶

Classes¶

LazyExpressionMatrix ¶

Attributes¶

shape: tuple[int, int] property ¶

obs_names: pd.Index | None property ¶

var_names: pd.Index | None property ¶

Functions¶

__init__(slaf_array: SLAFArray, table_name: str = 'expression', layer_name: str | None = None) ¶

mean(axis: int | None = None, fragments: bool | None = None) -> float | np.ndarray ¶

sum(axis: int | None = None, fragments: bool | None = None) -> float | np.ndarray ¶

var(axis: int | None = None) -> float | np.ndarray ¶

std(axis: int | None = None) -> float | np.ndarray ¶

toarray() -> np.ndarray ¶

compute(fragments: bool | None = None) -> scipy.sparse.csr_matrix ¶

LazyDictionaryViewMixin ¶

Functions¶

keys() -> list[str] ¶

LazyLayersView ¶

Functions¶

__init__(lazy_adata: LazyAnnData) ¶

keys() -> list[str] ¶

LazyMetadataViewMixin ¶

Functions¶

keys() -> list[str] ¶

LazyObsView ¶

Attributes¶

shape: tuple[int, int] property ¶

Functions¶

__init__(lazy_adata: LazyAnnData) ¶

LazyVarView ¶

Attributes¶

shape: tuple[int, int] property ¶

Functions¶

__init__(lazy_adata: LazyAnnData) ¶

LazyObsmView ¶

Attributes¶

shape: tuple[int, int] property ¶

Functions¶

__init__(lazy_adata: LazyAnnData) ¶

LazyVarmView ¶

Attributes¶

shape: tuple[int, int] property ¶

Functions¶

__init__(lazy_adata: LazyAnnData) ¶

LazyUnsView ¶

Functions¶

__init__(lazy_adata: LazyAnnData) ¶

keys() -> list[str] ¶

LazyAnnData ¶

Attributes¶

layers: LazyLayersView property ¶

obs: LazyObsView property ¶

var: LazyVarView property ¶

obsm: LazyObsmView property ¶

varm: LazyVarmView property ¶

uns: LazyUnsView property ¶

X: LazyExpressionMatrix property ¶

obs_deprecated: pd.DataFrame property ¶

var_deprecated: pd.DataFrame property ¶

obs_names: pd.Index property ¶

var_names: pd.Index property ¶

n_obs: int property ¶

n_vars: int property ¶

shape: tuple[int, int] property ¶

Functions¶

__init__(slaf_array: SLAFArray, backend: str = 'auto') ¶

copy() -> LazyAnnData ¶

to_memory() -> scipy.sparse.spmatrix ¶

write(filename: str) ¶

get_expression_data() -> scipy.sparse.csr_matrix ¶

compute() -> sc.AnnData ¶

Functions¶

read_slaf(filename: str, backend: str = 'auto') -> LazyAnnData ¶

Scanpy Integration¶

slaf.integrations.scanpy ¶

Classes¶

LazyPreprocessing ¶

`slaf.integrations.anndata` ¶

`LazyExpressionMatrix` ¶

`shape: tuple[int, int]` `property` ¶

`obs_names: pd.Index | None` `property` ¶

`var_names: pd.Index | None` `property` ¶

`init(slaf_array: SLAFArray, table_name: str = 'expression', layer_name: str | None = None)` ¶

`mean(axis: int | None = None, fragments: bool | None = None) -> float | np.ndarray` ¶

`sum(axis: int | None = None, fragments: bool | None = None) -> float | np.ndarray` ¶

`var(axis: int | None = None) -> float | np.ndarray` ¶

`std(axis: int | None = None) -> float | np.ndarray` ¶

`toarray() -> np.ndarray` ¶

`compute(fragments: bool | None = None) -> scipy.sparse.csr_matrix` ¶

`LazyDictionaryViewMixin` ¶

`keys() -> list[str]` ¶

`LazyLayersView` ¶

`init(lazy_adata: LazyAnnData)` ¶

`keys() -> list[str]` ¶

`LazyMetadataViewMixin` ¶

`keys() -> list[str]` ¶

`LazyObsView` ¶

`shape: tuple[int, int]` `property` ¶

`init(lazy_adata: LazyAnnData)` ¶

`LazyVarView` ¶

`shape: tuple[int, int]` `property` ¶

`init(lazy_adata: LazyAnnData)` ¶

`LazyObsmView` ¶

`shape: tuple[int, int]` `property` ¶

`init(lazy_adata: LazyAnnData)` ¶

`LazyVarmView` ¶

`shape: tuple[int, int]` `property` ¶

`init(lazy_adata: LazyAnnData)` ¶

`LazyUnsView` ¶

`init(lazy_adata: LazyAnnData)` ¶

`keys() -> list[str]` ¶

`LazyAnnData` ¶

`layers: LazyLayersView` `property` ¶

`obs: LazyObsView` `property` ¶

`var: LazyVarView` `property` ¶

`obsm: LazyObsmView` `property` ¶

`varm: LazyVarmView` `property` ¶

`uns: LazyUnsView` `property` ¶

`X: LazyExpressionMatrix` `property` ¶

`obs_deprecated: pd.DataFrame` `property` ¶

`var_deprecated: pd.DataFrame` `property` ¶

`obs_names: pd.Index` `property` ¶

`var_names: pd.Index` `property` ¶

`n_obs: int` `property` ¶

`n_vars: int` `property` ¶

`shape: tuple[int, int]` `property` ¶

`init(slaf_array: SLAFArray, backend: str = 'auto')` ¶

`copy() -> LazyAnnData` ¶

`to_memory() -> scipy.sparse.spmatrix` ¶

`write(filename: str)` ¶

`get_expression_data() -> scipy.sparse.csr_matrix` ¶

`compute() -> sc.AnnData` ¶

`read_slaf(filename: str, backend: str = 'auto') -> LazyAnnData` ¶

`slaf.integrations.scanpy` ¶

`LazyPreprocessing` ¶

`calculate_qc_metrics(adata: LazyAnnData, percent_top: int | list | None = None, log1p: bool = True, inplace: bool = True) -> tuple | None` `staticmethod` ¶

`filter_cells(adata: LazyAnnData, min_counts: int | None = None, min_genes: int | None = None, max_counts: int | None = None, max_genes: int | None = None, inplace: bool = True) -> LazyAnnData | None` `staticmethod` ¶

`filter_genes(adata: LazyAnnData, min_counts: int | None = None, min_cells: int | None = None, max_counts: int | None = None, max_cells: int | None = None, inplace: bool = True) -> LazyAnnData | None` `staticmethod` ¶

`normalize_total(adata: LazyAnnData, target_sum: float | None = 10000.0, inplace: bool = True, fragments: bool | None = None) -> LazyAnnData | None` `staticmethod` ¶

`log1p(adata: LazyAnnData, inplace: bool = True, fragments: bool | None = None) -> LazyAnnData | None` `staticmethod` ¶

`highly_variable_genes(adata: LazyAnnData, min_mean: float = 0.0125, max_mean: float = 3, min_disp: float = 0.5, max_disp: float = np.inf, n_top_genes: int | None = None, inplace: bool = True) -> pd.DataFrame | None` `staticmethod` ¶

`apply_transformations(adata: LazyAnnData, transformations: list | None = None) -> LazyAnnData` ¶

`clear_transformations(adata: LazyAnnData, inplace: bool = False) -> LazyAnnData | None` ¶